diff options
Diffstat (limited to 'target/linux/patches')
| -rw-r--r-- | target/linux/patches/4.1.10/realtime.patch | 46477 |
1 files changed, 45133 insertions, 1344 deletions
diff --git a/target/linux/patches/4.1.10/realtime.patch b/target/linux/patches/4.1.10/realtime.patch index 9b5b92ee1..e9d7af67d 100644 --- a/target/linux/patches/4.1.10/realtime.patch +++ b/target/linux/patches/4.1.10/realtime.patch @@ -1,6 +1,6 @@ -diff -Nur linux-4.1.6.orig/arch/alpha/mm/fault.c linux-4.1.6/arch/alpha/mm/fault.c ---- linux-4.1.6.orig/arch/alpha/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/alpha/mm/fault.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/alpha/mm/fault.c linux-4.1.10/arch/alpha/mm/fault.c +--- linux-4.1.10.orig/arch/alpha/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/alpha/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -23,8 +23,7 @@ #include <linux/smp.h> #include <linux/interrupt.h> @@ -20,9 +20,9 @@ diff -Nur linux-4.1.6.orig/arch/alpha/mm/fault.c linux-4.1.6/arch/alpha/mm/fault goto no_context; #ifdef CONFIG_ALPHA_LARGE_VMALLOC -diff -Nur linux-4.1.6.orig/arch/arc/include/asm/futex.h linux-4.1.6/arch/arc/include/asm/futex.h ---- linux-4.1.6.orig/arch/arc/include/asm/futex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arc/include/asm/futex.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arc/include/asm/futex.h linux-4.1.10/arch/arc/include/asm/futex.h +--- linux-4.1.10.orig/arch/arc/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arc/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200 @@ -53,7 +53,7 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; @@ -68,9 +68,9 @@ diff -Nur linux-4.1.6.orig/arch/arc/include/asm/futex.h linux-4.1.6/arch/arc/inc *uval = val; return val; -diff -Nur linux-4.1.6.orig/arch/arc/mm/fault.c linux-4.1.6/arch/arc/mm/fault.c ---- linux-4.1.6.orig/arch/arc/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arc/mm/fault.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arc/mm/fault.c linux-4.1.10/arch/arc/mm/fault.c +--- linux-4.1.10.orig/arch/arc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -86,7 +86,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -80,9 +80,9 @@ diff -Nur linux-4.1.6.orig/arch/arc/mm/fault.c linux-4.1.6/arch/arc/mm/fault.c goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.6/arch/arm/include/asm/cmpxchg.h ---- linux-4.1.6.orig/arch/arm/include/asm/cmpxchg.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/include/asm/cmpxchg.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.10/arch/arm/include/asm/cmpxchg.h +--- linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/cmpxchg.h 2015-10-07 18:00:07.000000000 +0200 @@ -129,6 +129,8 @@ #else /* min ARCH >= ARMv6 */ @@ -92,9 +92,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.6/arch/arm/i extern void __bad_cmpxchg(volatile void *ptr, int size); /* -diff -Nur linux-4.1.6.orig/arch/arm/include/asm/futex.h linux-4.1.6/arch/arm/include/asm/futex.h ---- linux-4.1.6.orig/arch/arm/include/asm/futex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/include/asm/futex.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/include/asm/futex.h linux-4.1.10/arch/arm/include/asm/futex.h +--- linux-4.1.10.orig/arch/arm/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200 @@ -93,6 +93,7 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; @@ -136,9 +136,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/include/asm/futex.h linux-4.1.6/arch/arm/inc if (!ret) { switch (cmp) { -diff -Nur linux-4.1.6.orig/arch/arm/include/asm/switch_to.h linux-4.1.6/arch/arm/include/asm/switch_to.h ---- linux-4.1.6.orig/arch/arm/include/asm/switch_to.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/include/asm/switch_to.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/include/asm/switch_to.h linux-4.1.10/arch/arm/include/asm/switch_to.h +--- linux-4.1.10.orig/arch/arm/include/asm/switch_to.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/switch_to.h 2015-10-07 18:00:07.000000000 +0200 @@ -3,6 +3,13 @@ #include <linux/thread_info.h> @@ -161,9 +161,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/include/asm/switch_to.h linux-4.1.6/arch/arm last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ } while (0) -diff -Nur linux-4.1.6.orig/arch/arm/include/asm/thread_info.h linux-4.1.6/arch/arm/include/asm/thread_info.h ---- linux-4.1.6.orig/arch/arm/include/asm/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/include/asm/thread_info.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/include/asm/thread_info.h linux-4.1.10/arch/arm/include/asm/thread_info.h +--- linux-4.1.10.orig/arch/arm/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 @@ -50,6 +50,7 @@ struct thread_info { unsigned long flags; /* low level flags */ @@ -188,9 +188,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/include/asm/thread_info.h linux-4.1.6/arch/a #define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) -diff -Nur linux-4.1.6.orig/arch/arm/Kconfig linux-4.1.6/arch/arm/Kconfig ---- linux-4.1.6.orig/arch/arm/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/Kconfig 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/Kconfig linux-4.1.10/arch/arm/Kconfig +--- linux-4.1.10.orig/arch/arm/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -31,7 +31,7 @@ select HARDIRQS_SW_RESEND select HAVE_ARCH_AUDITSYSCALL if (AEABI && !OABI_COMPAT) @@ -208,9 +208,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/Kconfig linux-4.1.6/arch/arm/Kconfig select HAVE_RCU_TABLE_FREE if (SMP && ARM_LPAE) select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_SYSCALL_TRACEPOINTS -diff -Nur linux-4.1.6.orig/arch/arm/kernel/asm-offsets.c linux-4.1.6/arch/arm/kernel/asm-offsets.c ---- linux-4.1.6.orig/arch/arm/kernel/asm-offsets.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/asm-offsets.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c linux-4.1.10/arch/arm/kernel/asm-offsets.c +--- linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 @@ -65,6 +65,7 @@ BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); @@ -219,9 +219,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/asm-offsets.c linux-4.1.6/arch/arm/ke DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_TASK, offsetof(struct thread_info, task)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); -diff -Nur linux-4.1.6.orig/arch/arm/kernel/entry-armv.S linux-4.1.6/arch/arm/kernel/entry-armv.S ---- linux-4.1.6.orig/arch/arm/kernel/entry-armv.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/entry-armv.S 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/entry-armv.S linux-4.1.10/arch/arm/kernel/entry-armv.S +--- linux-4.1.10.orig/arch/arm/kernel/entry-armv.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/entry-armv.S 2015-10-07 18:00:07.000000000 +0200 @@ -208,11 +208,18 @@ #ifdef CONFIG_PREEMPT get_thread_info tsk @@ -252,9 +252,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/entry-armv.S linux-4.1.6/arch/arm/ker reteq r8 @ go again b 1b #endif -diff -Nur linux-4.1.6.orig/arch/arm/kernel/process.c linux-4.1.6/arch/arm/kernel/process.c ---- linux-4.1.6.orig/arch/arm/kernel/process.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/process.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/process.c linux-4.1.10/arch/arm/kernel/process.c +--- linux-4.1.10.orig/arch/arm/kernel/process.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/process.c 2015-10-07 18:00:07.000000000 +0200 @@ -290,6 +290,30 @@ } @@ -286,9 +286,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/process.c linux-4.1.6/arch/arm/kernel #ifdef CONFIG_KUSER_HELPERS /* * The vectors page is always readable from user space for the -diff -Nur linux-4.1.6.orig/arch/arm/kernel/signal.c linux-4.1.6/arch/arm/kernel/signal.c ---- linux-4.1.6.orig/arch/arm/kernel/signal.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/signal.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/signal.c linux-4.1.10/arch/arm/kernel/signal.c +--- linux-4.1.10.orig/arch/arm/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/signal.c 2015-10-07 18:00:07.000000000 +0200 @@ -563,7 +563,8 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) { @@ -299,9 +299,31 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/signal.c linux-4.1.6/arch/arm/kernel/ schedule(); } else { if (unlikely(!user_mode(regs))) -diff -Nur linux-4.1.6.orig/arch/arm/kernel/unwind.c linux-4.1.6/arch/arm/kernel/unwind.c ---- linux-4.1.6.orig/arch/arm/kernel/unwind.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/unwind.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/smp.c linux-4.1.10/arch/arm/kernel/smp.c +--- linux-4.1.10.orig/arch/arm/kernel/smp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/smp.c 2015-10-07 18:00:07.000000000 +0200 +@@ -213,8 +213,6 @@ + flush_cache_louis(); + local_flush_tlb_all(); + +- clear_tasks_mm_cpumask(cpu); +- + return 0; + } + +@@ -230,6 +228,9 @@ + pr_err("CPU%u: cpu didn't die\n", cpu); + return; + } ++ ++ clear_tasks_mm_cpumask(cpu); ++ + pr_notice("CPU%u: shutdown\n", cpu); + + /* +diff -Nur linux-4.1.10.orig/arch/arm/kernel/unwind.c linux-4.1.10/arch/arm/kernel/unwind.c +--- linux-4.1.10.orig/arch/arm/kernel/unwind.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/unwind.c 2015-10-07 18:00:07.000000000 +0200 @@ -93,7 +93,7 @@ static const struct unwind_idx *__origin_unwind_idx; extern const struct unwind_idx __stop_unwind_idx[]; @@ -353,9 +375,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/unwind.c linux-4.1.6/arch/arm/kernel/ kfree(tab); } -diff -Nur linux-4.1.6.orig/arch/arm/kvm/arm.c linux-4.1.6/arch/arm/kvm/arm.c ---- linux-4.1.6.orig/arch/arm/kvm/arm.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kvm/arm.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kvm/arm.c linux-4.1.10/arch/arm/kvm/arm.c +--- linux-4.1.10.orig/arch/arm/kvm/arm.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kvm/arm.c 2015-10-07 18:00:07.000000000 +0200 @@ -474,9 +474,9 @@ static void vcpu_pause(struct kvm_vcpu *vcpu) @@ -368,9 +390,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kvm/arm.c linux-4.1.6/arch/arm/kvm/arm.c } static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu) -diff -Nur linux-4.1.6.orig/arch/arm/kvm/psci.c linux-4.1.6/arch/arm/kvm/psci.c ---- linux-4.1.6.orig/arch/arm/kvm/psci.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kvm/psci.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kvm/psci.c linux-4.1.10/arch/arm/kvm/psci.c +--- linux-4.1.10.orig/arch/arm/kvm/psci.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kvm/psci.c 2015-10-07 18:00:07.000000000 +0200 @@ -68,7 +68,7 @@ { struct kvm *kvm = source_vcpu->kvm; @@ -389,9 +411,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kvm/psci.c linux-4.1.6/arch/arm/kvm/psci.c return PSCI_RET_SUCCESS; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.6/arch/arm/mach-exynos/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-exynos/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-exynos/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.10/arch/arm/mach-exynos/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-exynos/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -231,7 +231,7 @@ return (void __iomem *)(S5P_VA_SCU); } @@ -439,9 +461,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.6/arch/arm/m return pen_release != -1 ? ret : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.6/arch/arm/mach-hisi/platmcpm.c ---- linux-4.1.6.orig/arch/arm/mach-hisi/platmcpm.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-hisi/platmcpm.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.10/arch/arm/mach-hisi/platmcpm.c +--- linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-hisi/platmcpm.c 2015-10-07 18:00:07.000000000 +0200 @@ -57,7 +57,7 @@ static void __iomem *sysctrl, *fabric; @@ -544,9 +566,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.6/arch/arm/ma } static void __naked hip04_mcpm_power_up_setup(unsigned int affinity_level) -diff -Nur linux-4.1.6.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.6/arch/arm/mach-omap2/omap-smp.c ---- linux-4.1.6.orig/arch/arm/mach-omap2/omap-smp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-omap2/omap-smp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.10/arch/arm/mach-omap2/omap-smp.c +--- linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-omap2/omap-smp.c 2015-10-07 18:00:07.000000000 +0200 @@ -43,7 +43,7 @@ /* SCU base address */ static void __iomem *scu_base; @@ -585,9 +607,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.6/arch/arm/m return 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.6/arch/arm/mach-prima2/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-prima2/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-prima2/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.10/arch/arm/mach-prima2/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-prima2/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -22,7 +22,7 @@ static void __iomem *clk_base; @@ -626,9 +648,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.6/arch/arm/m return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.6/arch/arm/mach-qcom/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-qcom/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-qcom/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.10/arch/arm/mach-qcom/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-qcom/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -46,7 +46,7 @@ extern void secondary_startup_arm(void); @@ -667,9 +689,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.6/arch/arm/mac return ret; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-spear/platsmp.c linux-4.1.6/arch/arm/mach-spear/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-spear/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-spear/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c linux-4.1.10/arch/arm/mach-spear/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-spear/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -32,7 +32,7 @@ sync_cache_w(&pen_release); } @@ -708,9 +730,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-spear/platsmp.c linux-4.1.6/arch/arm/ma return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-sti/platsmp.c linux-4.1.6/arch/arm/mach-sti/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-sti/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-sti/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c linux-4.1.10/arch/arm/mach-sti/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-sti/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -34,7 +34,7 @@ sync_cache_w(&pen_release); } @@ -749,9 +771,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-sti/platsmp.c linux-4.1.6/arch/arm/mach return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.6/arch/arm/mach-ux500/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-ux500/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-ux500/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.10/arch/arm/mach-ux500/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-ux500/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -51,7 +51,7 @@ return NULL; } @@ -790,9 +812,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.6/arch/arm/ma return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mm/fault.c linux-4.1.6/arch/arm/mm/fault.c ---- linux-4.1.6.orig/arch/arm/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mm/fault.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mm/fault.c linux-4.1.10/arch/arm/mm/fault.c +--- linux-4.1.10.orig/arch/arm/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -276,7 +276,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -822,9 +844,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mm/fault.c linux-4.1.6/arch/arm/mm/fault.c do_bad_area(addr, fsr, regs); return 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mm/highmem.c linux-4.1.6/arch/arm/mm/highmem.c ---- linux-4.1.6.orig/arch/arm/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mm/highmem.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mm/highmem.c linux-4.1.10/arch/arm/mm/highmem.c +--- linux-4.1.10.orig/arch/arm/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -54,11 +54,13 @@ void *kmap_atomic(struct page *page) @@ -927,9 +949,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mm/highmem.c linux-4.1.6/arch/arm/mm/highmem + } +} +#endif -diff -Nur linux-4.1.6.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.6/arch/arm/plat-versatile/platsmp.c ---- linux-4.1.6.orig/arch/arm/plat-versatile/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/plat-versatile/platsmp.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.10/arch/arm/plat-versatile/platsmp.c +--- linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/plat-versatile/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -30,7 +30,7 @@ sync_cache_w(&pen_release); } @@ -968,9 +990,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.6/arch/ar return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm64/include/asm/futex.h linux-4.1.6/arch/arm64/include/asm/futex.h ---- linux-4.1.6.orig/arch/arm64/include/asm/futex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/include/asm/futex.h 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/futex.h linux-4.1.10/arch/arm64/include/asm/futex.h +--- linux-4.1.10.orig/arch/arm64/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200 @@ -58,7 +58,7 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; @@ -989,9 +1011,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/include/asm/futex.h linux-4.1.6/arch/arm64 if (!ret) { switch (cmp) { -diff -Nur linux-4.1.6.orig/arch/arm64/include/asm/thread_info.h linux-4.1.6/arch/arm64/include/asm/thread_info.h ---- linux-4.1.6.orig/arch/arm64/include/asm/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/include/asm/thread_info.h 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h linux-4.1.10/arch/arm64/include/asm/thread_info.h +--- linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 @@ -47,6 +47,7 @@ mm_segment_t addr_limit; /* address limit */ struct task_struct *task; /* main task structure */ @@ -1016,9 +1038,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/include/asm/thread_info.h linux-4.1.6/arch #define _TIF_NOHZ (1 << TIF_NOHZ) #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) -diff -Nur linux-4.1.6.orig/arch/arm64/Kconfig linux-4.1.6/arch/arm64/Kconfig ---- linux-4.1.6.orig/arch/arm64/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/Kconfig 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/Kconfig linux-4.1.10/arch/arm64/Kconfig +--- linux-4.1.10.orig/arch/arm64/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -69,8 +69,10 @@ select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP @@ -1030,9 +1052,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/Kconfig linux-4.1.6/arch/arm64/Kconfig select MODULES_USE_ELF_RELA select NO_BOOTMEM select OF -diff -Nur linux-4.1.6.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.6/arch/arm64/kernel/asm-offsets.c ---- linux-4.1.6.orig/arch/arm64/kernel/asm-offsets.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/kernel/asm-offsets.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.10/arch/arm64/kernel/asm-offsets.c +--- linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 @@ -35,6 +35,7 @@ BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); @@ -1041,9 +1063,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.6/arch/arm6 DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_TASK, offsetof(struct thread_info, task)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); -diff -Nur linux-4.1.6.orig/arch/arm64/kernel/entry.S linux-4.1.6/arch/arm64/kernel/entry.S ---- linux-4.1.6.orig/arch/arm64/kernel/entry.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/kernel/entry.S 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/kernel/entry.S linux-4.1.10/arch/arm64/kernel/entry.S +--- linux-4.1.10.orig/arch/arm64/kernel/entry.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/kernel/entry.S 2015-10-07 18:00:07.000000000 +0200 @@ -367,11 +367,16 @@ #ifdef CONFIG_PREEMPT get_thread_info tsk @@ -1080,9 +1102,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/kernel/entry.S linux-4.1.6/arch/arm64/kern /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */ ldr x2, [sp, #S_PSTATE] mov x0, sp // 'regs' -diff -Nur linux-4.1.6.orig/arch/arm64/kernel/perf_event.c linux-4.1.6/arch/arm64/kernel/perf_event.c ---- linux-4.1.6.orig/arch/arm64/kernel/perf_event.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/kernel/perf_event.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/kernel/perf_event.c linux-4.1.10/arch/arm64/kernel/perf_event.c +--- linux-4.1.10.orig/arch/arm64/kernel/perf_event.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/kernel/perf_event.c 2015-10-07 18:00:07.000000000 +0200 @@ -488,7 +488,7 @@ } @@ -1092,9 +1114,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/kernel/perf_event.c linux-4.1.6/arch/arm64 "arm-pmu", armpmu); if (err) { pr_err("unable to request IRQ%d for ARM PMU counters\n", -diff -Nur linux-4.1.6.orig/arch/arm64/mm/fault.c linux-4.1.6/arch/arm64/mm/fault.c ---- linux-4.1.6.orig/arch/arm64/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/mm/fault.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/mm/fault.c linux-4.1.10/arch/arm64/mm/fault.c +--- linux-4.1.10.orig/arch/arm64/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -211,7 +211,7 @@ * If we're in an interrupt or have no user context, we must not take * the fault. @@ -1104,9 +1126,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/mm/fault.c linux-4.1.6/arch/arm64/mm/fault goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/avr32/include/asm/uaccess.h linux-4.1.6/arch/avr32/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/avr32/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/avr32/include/asm/uaccess.h 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h linux-4.1.10/arch/avr32/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/avr32/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -97,7 +97,8 @@ * @x: Value to copy to user space. * @ptr: Destination address, in user space. @@ -1147,9 +1169,9 @@ diff -Nur linux-4.1.6.orig/arch/avr32/include/asm/uaccess.h linux-4.1.6/arch/avr * * This macro copies a single simple variable from user space to kernel * space. It supports simple types like char and int, but not larger -diff -Nur linux-4.1.6.orig/arch/avr32/mm/fault.c linux-4.1.6/arch/avr32/mm/fault.c ---- linux-4.1.6.orig/arch/avr32/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/avr32/mm/fault.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/avr32/mm/fault.c linux-4.1.10/arch/avr32/mm/fault.c +--- linux-4.1.10.orig/arch/avr32/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/avr32/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -14,11 +14,11 @@ #include <linux/pagemap.h> #include <linux/kdebug.h> @@ -1172,9 +1194,9 @@ diff -Nur linux-4.1.6.orig/arch/avr32/mm/fault.c linux-4.1.6/arch/avr32/mm/fault goto no_context; local_irq_enable(); -diff -Nur linux-4.1.6.orig/arch/cris/mm/fault.c linux-4.1.6/arch/cris/mm/fault.c ---- linux-4.1.6.orig/arch/cris/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/cris/mm/fault.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/cris/mm/fault.c linux-4.1.10/arch/cris/mm/fault.c +--- linux-4.1.10.orig/arch/cris/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/cris/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -8,7 +8,7 @@ #include <linux/interrupt.h> #include <linux/module.h> @@ -1198,9 +1220,9 @@ diff -Nur linux-4.1.6.orig/arch/cris/mm/fault.c linux-4.1.6/arch/cris/mm/fault.c goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/frv/mm/fault.c linux-4.1.6/arch/frv/mm/fault.c ---- linux-4.1.6.orig/arch/frv/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/frv/mm/fault.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/frv/mm/fault.c linux-4.1.10/arch/frv/mm/fault.c +--- linux-4.1.10.orig/arch/frv/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/frv/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -19,9 +19,9 @@ #include <linux/kernel.h> #include <linux/ptrace.h> @@ -1221,9 +1243,9 @@ diff -Nur linux-4.1.6.orig/arch/frv/mm/fault.c linux-4.1.6/arch/frv/mm/fault.c goto no_context; if (user_mode(__frame)) -diff -Nur linux-4.1.6.orig/arch/frv/mm/highmem.c linux-4.1.6/arch/frv/mm/highmem.c ---- linux-4.1.6.orig/arch/frv/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/frv/mm/highmem.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/frv/mm/highmem.c linux-4.1.10/arch/frv/mm/highmem.c +--- linux-4.1.10.orig/arch/frv/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/frv/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -42,6 +42,7 @@ unsigned long paddr; int type; @@ -1239,9 +1261,9 @@ diff -Nur linux-4.1.6.orig/arch/frv/mm/highmem.c linux-4.1.6/arch/frv/mm/highmem + preempt_enable(); } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.6/arch/hexagon/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/hexagon/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/hexagon/include/asm/uaccess.h 2015-09-08 23:49:04.386279854 +0200 +diff -Nur linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.10/arch/hexagon/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/hexagon/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -36,7 +36,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -1252,9 +1274,9 @@ diff -Nur linux-4.1.6.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.6/arch/h * * Checks if a pointer to a block of memory in user space is valid. * -diff -Nur linux-4.1.6.orig/arch/ia64/mm/fault.c linux-4.1.6/arch/ia64/mm/fault.c ---- linux-4.1.6.orig/arch/ia64/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/ia64/mm/fault.c 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/ia64/mm/fault.c linux-4.1.10/arch/ia64/mm/fault.c +--- linux-4.1.10.orig/arch/ia64/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/ia64/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -11,10 +11,10 @@ #include <linux/kprobes.h> #include <linux/kdebug.h> @@ -1276,9 +1298,9 @@ diff -Nur linux-4.1.6.orig/arch/ia64/mm/fault.c linux-4.1.6/arch/ia64/mm/fault.c goto no_context; #ifdef CONFIG_VIRTUAL_MEM_MAP -diff -Nur linux-4.1.6.orig/arch/Kconfig linux-4.1.6/arch/Kconfig ---- linux-4.1.6.orig/arch/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/Kconfig 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/Kconfig linux-4.1.10/arch/Kconfig +--- linux-4.1.10.orig/arch/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -6,6 +6,7 @@ tristate "OProfile system profiling" depends on PROFILING @@ -1287,9 +1309,9 @@ diff -Nur linux-4.1.6.orig/arch/Kconfig linux-4.1.6/arch/Kconfig select RING_BUFFER select RING_BUFFER_ALLOW_SWAP help -diff -Nur linux-4.1.6.orig/arch/m32r/include/asm/uaccess.h linux-4.1.6/arch/m32r/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/m32r/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/m32r/include/asm/uaccess.h 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h linux-4.1.10/arch/m32r/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/m32r/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -91,7 +91,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -1390,9 +1412,9 @@ diff -Nur linux-4.1.6.orig/arch/m32r/include/asm/uaccess.h linux-4.1.6/arch/m32r * * Get the size of a NUL-terminated string in user space. * -diff -Nur linux-4.1.6.orig/arch/m32r/mm/fault.c linux-4.1.6/arch/m32r/mm/fault.c ---- linux-4.1.6.orig/arch/m32r/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/m32r/mm/fault.c 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/m32r/mm/fault.c linux-4.1.10/arch/m32r/mm/fault.c +--- linux-4.1.10.orig/arch/m32r/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/m32r/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -24,9 +24,9 @@ #include <linux/vt_kern.h> /* For unblank_screen() */ #include <linux/highmem.h> @@ -1418,9 +1440,9 @@ diff -Nur linux-4.1.6.orig/arch/m32r/mm/fault.c linux-4.1.6/arch/m32r/mm/fault.c goto bad_area_nosemaphore; if (error_code & ACE_USERMODE) -diff -Nur linux-4.1.6.orig/arch/m68k/mm/fault.c linux-4.1.6/arch/m68k/mm/fault.c ---- linux-4.1.6.orig/arch/m68k/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/m68k/mm/fault.c 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/m68k/mm/fault.c linux-4.1.10/arch/m68k/mm/fault.c +--- linux-4.1.10.orig/arch/m68k/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/m68k/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -10,10 +10,10 @@ #include <linux/ptrace.h> #include <linux/interrupt.h> @@ -1442,9 +1464,9 @@ diff -Nur linux-4.1.6.orig/arch/m68k/mm/fault.c linux-4.1.6/arch/m68k/mm/fault.c goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/metag/mm/fault.c linux-4.1.6/arch/metag/mm/fault.c ---- linux-4.1.6.orig/arch/metag/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/metag/mm/fault.c 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/metag/mm/fault.c linux-4.1.10/arch/metag/mm/fault.c +--- linux-4.1.10.orig/arch/metag/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/metag/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -105,7 +105,7 @@ mm = tsk->mm; @@ -1454,9 +1476,9 @@ diff -Nur linux-4.1.6.orig/arch/metag/mm/fault.c linux-4.1.6/arch/metag/mm/fault goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/metag/mm/highmem.c linux-4.1.6/arch/metag/mm/highmem.c ---- linux-4.1.6.orig/arch/metag/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/metag/mm/highmem.c 2015-09-08 23:49:04.662249253 +0200 +diff -Nur linux-4.1.10.orig/arch/metag/mm/highmem.c linux-4.1.10/arch/metag/mm/highmem.c +--- linux-4.1.10.orig/arch/metag/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/metag/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -43,7 +43,7 @@ unsigned long vaddr; int type; @@ -1482,9 +1504,9 @@ diff -Nur linux-4.1.6.orig/arch/metag/mm/highmem.c linux-4.1.6/arch/metag/mm/hig pagefault_disable(); type = kmap_atomic_idx_push(); -diff -Nur linux-4.1.6.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.6/arch/microblaze/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/microblaze/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/microblaze/include/asm/uaccess.h 2015-09-08 23:49:04.662249253 +0200 +diff -Nur linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.10/arch/microblaze/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/microblaze/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -178,7 +178,8 @@ * @x: Variable to store result. * @ptr: Source address, in user space. @@ -1505,9 +1527,9 @@ diff -Nur linux-4.1.6.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.6/arc * * This macro copies a single simple value from kernel space to user * space. It supports simple types like char and int, but not larger -diff -Nur linux-4.1.6.orig/arch/microblaze/mm/fault.c linux-4.1.6/arch/microblaze/mm/fault.c ---- linux-4.1.6.orig/arch/microblaze/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/microblaze/mm/fault.c 2015-09-08 23:49:04.670248364 +0200 +diff -Nur linux-4.1.10.orig/arch/microblaze/mm/fault.c linux-4.1.10/arch/microblaze/mm/fault.c +--- linux-4.1.10.orig/arch/microblaze/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/microblaze/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -107,14 +107,14 @@ if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) is_write = 0; @@ -1527,9 +1549,9 @@ diff -Nur linux-4.1.6.orig/arch/microblaze/mm/fault.c linux-4.1.6/arch/microblaz pr_emerg("r15 = %lx MSR = %lx\n", regs->r15, regs->msr); die("Weird page fault", regs, SIGSEGV); -diff -Nur linux-4.1.6.orig/arch/microblaze/mm/highmem.c linux-4.1.6/arch/microblaze/mm/highmem.c ---- linux-4.1.6.orig/arch/microblaze/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/microblaze/mm/highmem.c 2015-09-08 23:49:04.670248364 +0200 +diff -Nur linux-4.1.10.orig/arch/microblaze/mm/highmem.c linux-4.1.10/arch/microblaze/mm/highmem.c +--- linux-4.1.10.orig/arch/microblaze/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/microblaze/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -37,7 +37,7 @@ unsigned long vaddr; int idx, type; @@ -1554,9 +1576,9 @@ diff -Nur linux-4.1.6.orig/arch/microblaze/mm/highmem.c linux-4.1.6/arch/microbl + preempt_enable(); } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/mips/include/asm/uaccess.h linux-4.1.6/arch/mips/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/mips/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/include/asm/uaccess.h 2015-09-08 23:49:04.670248364 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/include/asm/uaccess.h linux-4.1.10/arch/mips/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/mips/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -103,7 +103,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -1707,9 +1729,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/include/asm/uaccess.h linux-4.1.6/arch/mips * * Get the size of a NUL-terminated string in user space. * -diff -Nur linux-4.1.6.orig/arch/mips/Kconfig linux-4.1.6/arch/mips/Kconfig ---- linux-4.1.6.orig/arch/mips/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/Kconfig 2015-09-08 23:49:04.670248364 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/Kconfig linux-4.1.10/arch/mips/Kconfig +--- linux-4.1.10.orig/arch/mips/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -2366,7 +2366,7 @@ # config HIGHMEM @@ -1719,9 +1741,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/Kconfig linux-4.1.6/arch/mips/Kconfig config CPU_SUPPORTS_HIGHMEM bool -diff -Nur linux-4.1.6.orig/arch/mips/kernel/signal-common.h linux-4.1.6/arch/mips/kernel/signal-common.h ---- linux-4.1.6.orig/arch/mips/kernel/signal-common.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/kernel/signal-common.h 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/kernel/signal-common.h linux-4.1.10/arch/mips/kernel/signal-common.h +--- linux-4.1.10.orig/arch/mips/kernel/signal-common.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/kernel/signal-common.h 2015-10-07 18:00:07.000000000 +0200 @@ -28,12 +28,7 @@ extern int fpcsr_pending(unsigned int __user *fpcsr); @@ -1737,9 +1759,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/kernel/signal-common.h linux-4.1.6/arch/mip +#define unlock_fpu_owner() ({ pagefault_enable(); preempt_enable(); }) #endif /* __SIGNAL_COMMON_H */ -diff -Nur linux-4.1.6.orig/arch/mips/mm/fault.c linux-4.1.6/arch/mips/mm/fault.c ---- linux-4.1.6.orig/arch/mips/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/mm/fault.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/mm/fault.c linux-4.1.10/arch/mips/mm/fault.c +--- linux-4.1.10.orig/arch/mips/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -21,10 +21,10 @@ #include <linux/module.h> #include <linux/kprobes.h> @@ -1761,9 +1783,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/mm/fault.c linux-4.1.6/arch/mips/mm/fault.c goto bad_area_nosemaphore; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/mips/mm/highmem.c linux-4.1.6/arch/mips/mm/highmem.c ---- linux-4.1.6.orig/arch/mips/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/mm/highmem.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/mm/highmem.c linux-4.1.10/arch/mips/mm/highmem.c +--- linux-4.1.10.orig/arch/mips/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -47,7 +47,7 @@ unsigned long vaddr; int idx, type; @@ -1797,9 +1819,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/mm/highmem.c linux-4.1.6/arch/mips/mm/highm pagefault_disable(); type = kmap_atomic_idx_push(); -diff -Nur linux-4.1.6.orig/arch/mips/mm/init.c linux-4.1.6/arch/mips/mm/init.c ---- linux-4.1.6.orig/arch/mips/mm/init.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/mm/init.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/mm/init.c linux-4.1.10/arch/mips/mm/init.c +--- linux-4.1.10.orig/arch/mips/mm/init.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/mm/init.c 2015-10-07 18:00:07.000000000 +0200 @@ -90,6 +90,7 @@ BUG_ON(Page_dcache_dirty(page)); @@ -1816,9 +1838,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/mm/init.c linux-4.1.6/arch/mips/mm/init.c } void copy_user_highpage(struct page *to, struct page *from, -diff -Nur linux-4.1.6.orig/arch/mn10300/include/asm/highmem.h linux-4.1.6/arch/mn10300/include/asm/highmem.h ---- linux-4.1.6.orig/arch/mn10300/include/asm/highmem.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mn10300/include/asm/highmem.h 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h linux-4.1.10/arch/mn10300/include/asm/highmem.h +--- linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mn10300/include/asm/highmem.h 2015-10-07 18:00:07.000000000 +0200 @@ -75,6 +75,7 @@ unsigned long vaddr; int idx, type; @@ -1843,9 +1865,9 @@ diff -Nur linux-4.1.6.orig/arch/mn10300/include/asm/highmem.h linux-4.1.6/arch/m } #endif /* __KERNEL__ */ -diff -Nur linux-4.1.6.orig/arch/mn10300/mm/fault.c linux-4.1.6/arch/mn10300/mm/fault.c ---- linux-4.1.6.orig/arch/mn10300/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mn10300/mm/fault.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mn10300/mm/fault.c linux-4.1.10/arch/mn10300/mm/fault.c +--- linux-4.1.10.orig/arch/mn10300/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mn10300/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -23,8 +23,8 @@ #include <linux/interrupt.h> #include <linux/init.h> @@ -1865,9 +1887,9 @@ diff -Nur linux-4.1.6.orig/arch/mn10300/mm/fault.c linux-4.1.6/arch/mn10300/mm/f goto no_context; if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR) -diff -Nur linux-4.1.6.orig/arch/nios2/mm/fault.c linux-4.1.6/arch/nios2/mm/fault.c ---- linux-4.1.6.orig/arch/nios2/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/nios2/mm/fault.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/nios2/mm/fault.c linux-4.1.10/arch/nios2/mm/fault.c +--- linux-4.1.10.orig/arch/nios2/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/nios2/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -77,7 +77,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -1877,9 +1899,9 @@ diff -Nur linux-4.1.6.orig/arch/nios2/mm/fault.c linux-4.1.6/arch/nios2/mm/fault goto bad_area_nosemaphore; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.6/arch/parisc/include/asm/cacheflush.h ---- linux-4.1.6.orig/arch/parisc/include/asm/cacheflush.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/parisc/include/asm/cacheflush.h 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.10/arch/parisc/include/asm/cacheflush.h +--- linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/parisc/include/asm/cacheflush.h 2015-10-07 18:00:07.000000000 +0200 @@ -142,6 +142,7 @@ static inline void *kmap_atomic(struct page *page) @@ -1896,9 +1918,9 @@ diff -Nur linux-4.1.6.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.6/arch } #define kmap_atomic_prot(page, prot) kmap_atomic(page) -diff -Nur linux-4.1.6.orig/arch/parisc/kernel/traps.c linux-4.1.6/arch/parisc/kernel/traps.c ---- linux-4.1.6.orig/arch/parisc/kernel/traps.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/parisc/kernel/traps.c 2015-09-08 23:49:04.678247479 +0200 +diff -Nur linux-4.1.10.orig/arch/parisc/kernel/traps.c linux-4.1.10/arch/parisc/kernel/traps.c +--- linux-4.1.10.orig/arch/parisc/kernel/traps.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/parisc/kernel/traps.c 2015-10-07 18:00:07.000000000 +0200 @@ -26,9 +26,9 @@ #include <linux/console.h> #include <linux/bug.h> @@ -1919,9 +1941,9 @@ diff -Nur linux-4.1.6.orig/arch/parisc/kernel/traps.c linux-4.1.6/arch/parisc/ke { pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC); parisc_terminate("Kernel Fault", regs, code, fault_address); -diff -Nur linux-4.1.6.orig/arch/parisc/mm/fault.c linux-4.1.6/arch/parisc/mm/fault.c ---- linux-4.1.6.orig/arch/parisc/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/parisc/mm/fault.c 2015-09-08 23:49:04.678247479 +0200 +diff -Nur linux-4.1.10.orig/arch/parisc/mm/fault.c linux-4.1.10/arch/parisc/mm/fault.c +--- linux-4.1.10.orig/arch/parisc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/parisc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -15,8 +15,8 @@ #include <linux/sched.h> #include <linux/interrupt.h> @@ -1941,9 +1963,9 @@ diff -Nur linux-4.1.6.orig/arch/parisc/mm/fault.c linux-4.1.6/arch/parisc/mm/fau goto no_context; tsk = current; -diff -Nur linux-4.1.6.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.6/arch/powerpc/include/asm/kvm_host.h ---- linux-4.1.6.orig/arch/powerpc/include/asm/kvm_host.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/include/asm/kvm_host.h 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.10/arch/powerpc/include/asm/kvm_host.h +--- linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/include/asm/kvm_host.h 2015-10-07 18:00:07.000000000 +0200 @@ -280,7 +280,7 @@ u8 in_guest; struct list_head runnable_threads; @@ -1962,9 +1984,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.6/arch/ struct kvmppc_vcore *vcore; int ret; int trap; -diff -Nur linux-4.1.6.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.6/arch/powerpc/include/asm/thread_info.h ---- linux-4.1.6.orig/arch/powerpc/include/asm/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/include/asm/thread_info.h 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.10/arch/powerpc/include/asm/thread_info.h +--- linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 @@ -42,6 +42,8 @@ int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, @@ -2011,9 +2033,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.6/ar /* Bits in local_flags */ /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */ -diff -Nur linux-4.1.6.orig/arch/powerpc/Kconfig linux-4.1.6/arch/powerpc/Kconfig ---- linux-4.1.6.orig/arch/powerpc/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/Kconfig 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/Kconfig linux-4.1.10/arch/powerpc/Kconfig +--- linux-4.1.10.orig/arch/powerpc/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -60,10 +60,11 @@ config RWSEM_GENERIC_SPINLOCK @@ -2044,9 +2066,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/Kconfig linux-4.1.6/arch/powerpc/Kconfig source kernel/Kconfig.hz source kernel/Kconfig.preempt -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.6/arch/powerpc/kernel/asm-offsets.c ---- linux-4.1.6.orig/arch/powerpc/kernel/asm-offsets.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/asm-offsets.c 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.10/arch/powerpc/kernel/asm-offsets.c +--- linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 @@ -160,6 +160,7 @@ DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags)); @@ -2055,9 +2077,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.6/arch/po DEFINE(TI_TASK, offsetof(struct thread_info, task)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/entry_32.S linux-4.1.6/arch/powerpc/kernel/entry_32.S ---- linux-4.1.6.orig/arch/powerpc/kernel/entry_32.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/entry_32.S 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S linux-4.1.10/arch/powerpc/kernel/entry_32.S +--- linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/entry_32.S 2015-10-07 18:00:07.000000000 +0200 @@ -813,7 +813,14 @@ cmpwi 0,r0,0 /* if non-zero, just restore regs and return */ bne restore @@ -2106,9 +2128,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/entry_32.S linux-4.1.6/arch/power bne- do_resched andi. r0,r9,_TIF_USER_WORK_MASK beq restore_user -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/entry_64.S linux-4.1.6/arch/powerpc/kernel/entry_64.S ---- linux-4.1.6.orig/arch/powerpc/kernel/entry_64.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/entry_64.S 2015-09-08 23:49:04.818231956 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S linux-4.1.10/arch/powerpc/kernel/entry_64.S +--- linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/entry_64.S 2015-10-07 18:00:07.000000000 +0200 @@ -636,7 +636,7 @@ #else beq restore @@ -2147,9 +2169,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/entry_64.S linux-4.1.6/arch/power bne 1b /* -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/irq.c linux-4.1.6/arch/powerpc/kernel/irq.c ---- linux-4.1.6.orig/arch/powerpc/kernel/irq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/irq.c 2015-09-08 23:49:04.818231956 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/irq.c linux-4.1.10/arch/powerpc/kernel/irq.c +--- linux-4.1.10.orig/arch/powerpc/kernel/irq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/irq.c 2015-10-07 18:00:07.000000000 +0200 @@ -614,6 +614,7 @@ } } @@ -2166,9 +2188,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/irq.c linux-4.1.6/arch/powerpc/ke irq_hw_number_t virq_to_hw(unsigned int virq) { -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/misc_32.S linux-4.1.6/arch/powerpc/kernel/misc_32.S ---- linux-4.1.6.orig/arch/powerpc/kernel/misc_32.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/misc_32.S 2015-09-08 23:49:04.818231956 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S linux-4.1.10/arch/powerpc/kernel/misc_32.S +--- linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/misc_32.S 2015-10-07 18:00:07.000000000 +0200 @@ -40,6 +40,7 @@ * We store the saved ksp_limit in the unused part * of the STACK_FRAME_OVERHEAD @@ -2185,9 +2207,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/misc_32.S linux-4.1.6/arch/powerp /* * void call_do_irq(struct pt_regs *regs, struct thread_info *irqtp); -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/misc_64.S linux-4.1.6/arch/powerpc/kernel/misc_64.S ---- linux-4.1.6.orig/arch/powerpc/kernel/misc_64.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/misc_64.S 2015-09-08 23:49:04.822231513 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S linux-4.1.10/arch/powerpc/kernel/misc_64.S +--- linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/misc_64.S 2015-10-07 18:00:07.000000000 +0200 @@ -29,6 +29,7 @@ .text @@ -2204,9 +2226,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/misc_64.S linux-4.1.6/arch/powerp _GLOBAL(call_do_irq) mflr r0 -diff -Nur linux-4.1.6.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.6/arch/powerpc/kvm/book3s_hv.c ---- linux-4.1.6.orig/arch/powerpc/kvm/book3s_hv.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kvm/book3s_hv.c 2015-09-08 23:49:04.826231069 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.10/arch/powerpc/kvm/book3s_hv.c +--- linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kvm/book3s_hv.c 2015-10-07 18:00:07.000000000 +0200 @@ -115,11 +115,11 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) { @@ -2282,9 +2304,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.6/arch/powerpc } } -diff -Nur linux-4.1.6.orig/arch/powerpc/kvm/Kconfig linux-4.1.6/arch/powerpc/kvm/Kconfig ---- linux-4.1.6.orig/arch/powerpc/kvm/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kvm/Kconfig 2015-09-08 23:49:04.822231513 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/Kconfig linux-4.1.10/arch/powerpc/kvm/Kconfig +--- linux-4.1.10.orig/arch/powerpc/kvm/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kvm/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -172,6 +172,7 @@ config KVM_MPIC bool "KVM in-kernel MPIC emulation" @@ -2293,9 +2315,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kvm/Kconfig linux-4.1.6/arch/powerpc/kvm select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQFD select HAVE_KVM_IRQ_ROUTING -diff -Nur linux-4.1.6.orig/arch/powerpc/mm/fault.c linux-4.1.6/arch/powerpc/mm/fault.c ---- linux-4.1.6.orig/arch/powerpc/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/mm/fault.c 2015-09-08 23:49:05.246184500 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/mm/fault.c linux-4.1.10/arch/powerpc/mm/fault.c +--- linux-4.1.10.orig/arch/powerpc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -33,13 +33,13 @@ #include <linux/ratelimit.h> #include <linux/context_tracking.h> @@ -2331,9 +2353,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/mm/fault.c linux-4.1.6/arch/powerpc/mm/f printk(KERN_EMERG "NIP = %lx MSR = %lx\n", regs->nip, regs->msr); die("Weird page fault", regs, SIGSEGV); -diff -Nur linux-4.1.6.orig/arch/powerpc/mm/highmem.c linux-4.1.6/arch/powerpc/mm/highmem.c ---- linux-4.1.6.orig/arch/powerpc/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/mm/highmem.c 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/mm/highmem.c linux-4.1.10/arch/powerpc/mm/highmem.c +--- linux-4.1.10.orig/arch/powerpc/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -34,7 +34,7 @@ unsigned long vaddr; int idx, type; @@ -2358,9 +2380,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/mm/highmem.c linux-4.1.6/arch/powerpc/mm + preempt_enable(); } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.6/arch/powerpc/platforms/ps3/device-init.c ---- linux-4.1.6.orig/arch/powerpc/platforms/ps3/device-init.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/platforms/ps3/device-init.c 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c +--- linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c 2015-10-07 18:00:07.000000000 +0200 @@ -752,7 +752,7 @@ } pr_debug("%s:%u: notification %s issued\n", __func__, __LINE__, op); @@ -2370,9 +2392,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.6/ dev->done.done || kthread_should_stop()); if (kthread_should_stop()) res = -EINTR; -diff -Nur linux-4.1.6.orig/arch/s390/include/asm/kvm_host.h linux-4.1.6/arch/s390/include/asm/kvm_host.h ---- linux-4.1.6.orig/arch/s390/include/asm/kvm_host.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/s390/include/asm/kvm_host.h 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h linux-4.1.10/arch/s390/include/asm/kvm_host.h +--- linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/s390/include/asm/kvm_host.h 2015-10-07 18:00:07.000000000 +0200 @@ -419,7 +419,7 @@ struct kvm_s390_local_interrupt { spinlock_t lock; @@ -2382,9 +2404,9 @@ diff -Nur linux-4.1.6.orig/arch/s390/include/asm/kvm_host.h linux-4.1.6/arch/s39 atomic_t *cpuflags; DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); struct kvm_s390_irq_payload irq; -diff -Nur linux-4.1.6.orig/arch/s390/include/asm/uaccess.h linux-4.1.6/arch/s390/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/s390/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/s390/include/asm/uaccess.h 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/s390/include/asm/uaccess.h linux-4.1.10/arch/s390/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/s390/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/s390/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -98,7 +98,8 @@ * @from: Source address, in user space. * @n: Number of bytes to copy. @@ -2435,9 +2457,9 @@ diff -Nur linux-4.1.6.orig/arch/s390/include/asm/uaccess.h linux-4.1.6/arch/s390 * * Get the size of a NUL-terminated string in user space. * -diff -Nur linux-4.1.6.orig/arch/s390/kvm/interrupt.c linux-4.1.6/arch/s390/kvm/interrupt.c ---- linux-4.1.6.orig/arch/s390/kvm/interrupt.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/s390/kvm/interrupt.c 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/s390/kvm/interrupt.c linux-4.1.10/arch/s390/kvm/interrupt.c +--- linux-4.1.10.orig/arch/s390/kvm/interrupt.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/s390/kvm/interrupt.c 2015-10-07 18:00:07.000000000 +0200 @@ -875,13 +875,13 @@ void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) @@ -2472,9 +2494,9 @@ diff -Nur linux-4.1.6.orig/arch/s390/kvm/interrupt.c linux-4.1.6/arch/s390/kvm/i spin_unlock(&li->lock); return rc; } -diff -Nur linux-4.1.6.orig/arch/s390/mm/fault.c linux-4.1.6/arch/s390/mm/fault.c ---- linux-4.1.6.orig/arch/s390/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/s390/mm/fault.c 2015-09-08 23:49:05.254183611 +0200 +diff -Nur linux-4.1.10.orig/arch/s390/mm/fault.c linux-4.1.10/arch/s390/mm/fault.c +--- linux-4.1.10.orig/arch/s390/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/s390/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -399,7 +399,7 @@ * user context. */ @@ -2484,9 +2506,9 @@ diff -Nur linux-4.1.6.orig/arch/s390/mm/fault.c linux-4.1.6/arch/s390/mm/fault.c goto out; address = trans_exc_code & __FAIL_ADDR_MASK; -diff -Nur linux-4.1.6.orig/arch/score/include/asm/uaccess.h linux-4.1.6/arch/score/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/score/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/score/include/asm/uaccess.h 2015-09-08 23:49:05.254183611 +0200 +diff -Nur linux-4.1.10.orig/arch/score/include/asm/uaccess.h linux-4.1.10/arch/score/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/score/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/score/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -36,7 +36,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -2537,9 +2559,9 @@ diff -Nur linux-4.1.6.orig/arch/score/include/asm/uaccess.h linux-4.1.6/arch/sco * * This macro copies a single simple variable from user space to kernel * space. It supports simple types like char and int, but not larger -diff -Nur linux-4.1.6.orig/arch/score/mm/fault.c linux-4.1.6/arch/score/mm/fault.c ---- linux-4.1.6.orig/arch/score/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/score/mm/fault.c 2015-09-08 23:49:05.254183611 +0200 +diff -Nur linux-4.1.10.orig/arch/score/mm/fault.c linux-4.1.10/arch/score/mm/fault.c +--- linux-4.1.10.orig/arch/score/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/score/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -34,6 +34,7 @@ #include <linux/string.h> #include <linux/types.h> @@ -2557,9 +2579,9 @@ diff -Nur linux-4.1.6.orig/arch/score/mm/fault.c linux-4.1.6/arch/score/mm/fault goto bad_area_nosemaphore; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/sh/kernel/irq.c linux-4.1.6/arch/sh/kernel/irq.c ---- linux-4.1.6.orig/arch/sh/kernel/irq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sh/kernel/irq.c 2015-09-08 23:49:05.254183611 +0200 +diff -Nur linux-4.1.10.orig/arch/sh/kernel/irq.c linux-4.1.10/arch/sh/kernel/irq.c +--- linux-4.1.10.orig/arch/sh/kernel/irq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sh/kernel/irq.c 2015-10-07 18:00:07.000000000 +0200 @@ -147,6 +147,7 @@ hardirq_ctx[cpu] = NULL; } @@ -2576,9 +2598,9 @@ diff -Nur linux-4.1.6.orig/arch/sh/kernel/irq.c linux-4.1.6/arch/sh/kernel/irq.c #else static inline void handle_one_irq(unsigned int irq) { -diff -Nur linux-4.1.6.orig/arch/sh/mm/fault.c linux-4.1.6/arch/sh/mm/fault.c ---- linux-4.1.6.orig/arch/sh/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sh/mm/fault.c 2015-09-08 23:49:05.258183169 +0200 +diff -Nur linux-4.1.10.orig/arch/sh/mm/fault.c linux-4.1.10/arch/sh/mm/fault.c +--- linux-4.1.10.orig/arch/sh/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sh/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -17,6 +17,7 @@ #include <linux/kprobes.h> #include <linux/perf_event.h> @@ -2599,9 +2621,9 @@ diff -Nur linux-4.1.6.orig/arch/sh/mm/fault.c linux-4.1.6/arch/sh/mm/fault.c bad_area_nosemaphore(regs, error_code, address); return; } -diff -Nur linux-4.1.6.orig/arch/sparc/Kconfig linux-4.1.6/arch/sparc/Kconfig ---- linux-4.1.6.orig/arch/sparc/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/Kconfig 2015-09-08 23:49:05.258183169 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/Kconfig linux-4.1.10/arch/sparc/Kconfig +--- linux-4.1.10.orig/arch/sparc/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -189,12 +189,10 @@ source kernel/Kconfig.hz @@ -2617,9 +2639,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/Kconfig linux-4.1.6/arch/sparc/Kconfig config GENERIC_HWEIGHT bool -diff -Nur linux-4.1.6.orig/arch/sparc/kernel/irq_64.c linux-4.1.6/arch/sparc/kernel/irq_64.c ---- linux-4.1.6.orig/arch/sparc/kernel/irq_64.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/kernel/irq_64.c 2015-09-08 23:49:05.258183169 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/kernel/irq_64.c linux-4.1.10/arch/sparc/kernel/irq_64.c +--- linux-4.1.10.orig/arch/sparc/kernel/irq_64.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/kernel/irq_64.c 2015-10-07 18:00:07.000000000 +0200 @@ -849,6 +849,7 @@ set_irq_regs(old_regs); } @@ -2636,9 +2658,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/kernel/irq_64.c linux-4.1.6/arch/sparc/ker #ifdef CONFIG_HOTPLUG_CPU void fixup_irqs(void) -diff -Nur linux-4.1.6.orig/arch/sparc/mm/fault_32.c linux-4.1.6/arch/sparc/mm/fault_32.c ---- linux-4.1.6.orig/arch/sparc/mm/fault_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/mm/fault_32.c 2015-09-08 23:49:05.258183169 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_32.c linux-4.1.10/arch/sparc/mm/fault_32.c +--- linux-4.1.10.orig/arch/sparc/mm/fault_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/fault_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -21,6 +21,7 @@ #include <linux/perf_event.h> #include <linux/interrupt.h> @@ -2664,9 +2686,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/mm/fault_32.c linux-4.1.6/arch/sparc/mm/fa goto no_context; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); -diff -Nur linux-4.1.6.orig/arch/sparc/mm/fault_64.c linux-4.1.6/arch/sparc/mm/fault_64.c ---- linux-4.1.6.orig/arch/sparc/mm/fault_64.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/mm/fault_64.c 2015-09-08 23:49:05.262182726 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_64.c linux-4.1.10/arch/sparc/mm/fault_64.c +--- linux-4.1.10.orig/arch/sparc/mm/fault_64.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/fault_64.c 2015-10-07 18:00:07.000000000 +0200 @@ -22,12 +22,12 @@ #include <linux/kdebug.h> #include <linux/percpu.h> @@ -2690,9 +2712,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/mm/fault_64.c linux-4.1.6/arch/sparc/mm/fa goto intr_or_no_mm; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); -diff -Nur linux-4.1.6.orig/arch/sparc/mm/highmem.c linux-4.1.6/arch/sparc/mm/highmem.c ---- linux-4.1.6.orig/arch/sparc/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/mm/highmem.c 2015-09-08 23:49:05.262182726 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/mm/highmem.c linux-4.1.10/arch/sparc/mm/highmem.c +--- linux-4.1.10.orig/arch/sparc/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -53,7 +53,7 @@ unsigned long vaddr; long idx, type; @@ -2717,9 +2739,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/mm/highmem.c linux-4.1.6/arch/sparc/mm/hig + preempt_enable(); } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/sparc/mm/init_64.c linux-4.1.6/arch/sparc/mm/init_64.c ---- linux-4.1.6.orig/arch/sparc/mm/init_64.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/mm/init_64.c 2015-09-08 23:49:05.262182726 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/mm/init_64.c linux-4.1.10/arch/sparc/mm/init_64.c +--- linux-4.1.10.orig/arch/sparc/mm/init_64.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/init_64.c 2015-10-07 18:00:07.000000000 +0200 @@ -2738,7 +2738,7 @@ struct mm_struct *mm = current->mm; struct tsb_config *tp; @@ -2729,9 +2751,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/mm/init_64.c linux-4.1.6/arch/sparc/mm/ini const struct exception_table_entry *entry; entry = search_exception_tables(regs->tpc); -diff -Nur linux-4.1.6.orig/arch/tile/include/asm/uaccess.h linux-4.1.6/arch/tile/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/tile/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/tile/include/asm/uaccess.h 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/tile/include/asm/uaccess.h linux-4.1.10/arch/tile/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/tile/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/tile/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -78,7 +78,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -2792,9 +2814,9 @@ diff -Nur linux-4.1.6.orig/arch/tile/include/asm/uaccess.h linux-4.1.6/arch/tile * * Copy data from user space to user space. Caller must check * the specified blocks with access_ok() before calling this function. -diff -Nur linux-4.1.6.orig/arch/tile/mm/fault.c linux-4.1.6/arch/tile/mm/fault.c ---- linux-4.1.6.orig/arch/tile/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/tile/mm/fault.c 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/tile/mm/fault.c linux-4.1.10/arch/tile/mm/fault.c +--- linux-4.1.10.orig/arch/tile/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/tile/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -354,9 +354,9 @@ /* @@ -2807,9 +2829,9 @@ diff -Nur linux-4.1.6.orig/arch/tile/mm/fault.c linux-4.1.6/arch/tile/mm/fault.c vma = NULL; /* happy compiler */ goto bad_area_nosemaphore; } -diff -Nur linux-4.1.6.orig/arch/tile/mm/highmem.c linux-4.1.6/arch/tile/mm/highmem.c ---- linux-4.1.6.orig/arch/tile/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/tile/mm/highmem.c 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/tile/mm/highmem.c linux-4.1.10/arch/tile/mm/highmem.c +--- linux-4.1.10.orig/arch/tile/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/tile/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -201,7 +201,7 @@ int idx, type; pte_t *pte; @@ -2827,9 +2849,9 @@ diff -Nur linux-4.1.6.orig/arch/tile/mm/highmem.c linux-4.1.6/arch/tile/mm/highm } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/um/kernel/trap.c linux-4.1.6/arch/um/kernel/trap.c ---- linux-4.1.6.orig/arch/um/kernel/trap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/um/kernel/trap.c 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/um/kernel/trap.c linux-4.1.10/arch/um/kernel/trap.c +--- linux-4.1.10.orig/arch/um/kernel/trap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/um/kernel/trap.c 2015-10-07 18:00:07.000000000 +0200 @@ -35,10 +35,10 @@ *code_out = SEGV_MAPERR; @@ -2843,9 +2865,9 @@ diff -Nur linux-4.1.6.orig/arch/um/kernel/trap.c linux-4.1.6/arch/um/kernel/trap goto out_nosemaphore; if (is_user) -diff -Nur linux-4.1.6.orig/arch/unicore32/mm/fault.c linux-4.1.6/arch/unicore32/mm/fault.c ---- linux-4.1.6.orig/arch/unicore32/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/unicore32/mm/fault.c 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/unicore32/mm/fault.c linux-4.1.10/arch/unicore32/mm/fault.c +--- linux-4.1.10.orig/arch/unicore32/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/unicore32/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -218,7 +218,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -2855,9 +2877,9 @@ diff -Nur linux-4.1.6.orig/arch/unicore32/mm/fault.c linux-4.1.6/arch/unicore32/ goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.6/arch/x86/crypto/aesni-intel_glue.c ---- linux-4.1.6.orig/arch/x86/crypto/aesni-intel_glue.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/crypto/aesni-intel_glue.c 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c +--- linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c 2015-10-07 18:00:07.000000000 +0200 @@ -382,14 +382,14 @@ err = blkcipher_walk_virt(desc, &walk); desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; @@ -2950,9 +2972,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.6/arch/x return err; } -diff -Nur linux-4.1.6.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.6/arch/x86/crypto/cast5_avx_glue.c ---- linux-4.1.6.orig/arch/x86/crypto/cast5_avx_glue.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/crypto/cast5_avx_glue.c 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c +--- linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c 2015-10-07 18:00:07.000000000 +0200 @@ -60,7 +60,7 @@ static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, bool enc) @@ -3032,9 +3054,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.6/arch/x86 if (walk.nbytes) { ctr_crypt_final(desc, &walk); err = blkcipher_walk_done(desc, &walk, 0); -diff -Nur linux-4.1.6.orig/arch/x86/crypto/glue_helper.c linux-4.1.6/arch/x86/crypto/glue_helper.c ---- linux-4.1.6.orig/arch/x86/crypto/glue_helper.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/crypto/glue_helper.c 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/crypto/glue_helper.c linux-4.1.10/arch/x86/crypto/glue_helper.c +--- linux-4.1.10.orig/arch/x86/crypto/glue_helper.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/crypto/glue_helper.c 2015-10-07 18:00:07.000000000 +0200 @@ -39,7 +39,7 @@ void *ctx = crypto_blkcipher_ctx(desc->tfm); const unsigned int bsize = 128 / 8; @@ -3150,9 +3172,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/crypto/glue_helper.c linux-4.1.6/arch/x86/cr return err; } EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit); -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/preempt.h linux-4.1.6/arch/x86/include/asm/preempt.h ---- linux-4.1.6.orig/arch/x86/include/asm/preempt.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/preempt.h 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/preempt.h linux-4.1.10/arch/x86/include/asm/preempt.h +--- linux-4.1.10.orig/arch/x86/include/asm/preempt.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/preempt.h 2015-10-07 18:00:07.000000000 +0200 @@ -82,17 +82,33 @@ * a decrement which hits zero means we have no preempt_count and should * reschedule. @@ -3188,9 +3210,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/preempt.h linux-4.1.6/arch/x86/i } #ifdef CONFIG_PREEMPT -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/signal.h linux-4.1.6/arch/x86/include/asm/signal.h ---- linux-4.1.6.orig/arch/x86/include/asm/signal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/signal.h 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/signal.h linux-4.1.10/arch/x86/include/asm/signal.h +--- linux-4.1.10.orig/arch/x86/include/asm/signal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/signal.h 2015-10-07 18:00:07.000000000 +0200 @@ -23,6 +23,19 @@ unsigned long sig[_NSIG_WORDS]; } sigset_t; @@ -3211,9 +3233,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/signal.h linux-4.1.6/arch/x86/in #ifndef CONFIG_COMPAT typedef sigset_t compat_sigset_t; #endif -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/stackprotector.h linux-4.1.6/arch/x86/include/asm/stackprotector.h ---- linux-4.1.6.orig/arch/x86/include/asm/stackprotector.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/stackprotector.h 2015-09-08 23:49:05.298178732 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h linux-4.1.10/arch/x86/include/asm/stackprotector.h +--- linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/stackprotector.h 2015-10-07 18:00:07.000000000 +0200 @@ -57,7 +57,7 @@ */ static __always_inline void boot_init_stack_canary(void) @@ -3240,9 +3262,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/stackprotector.h linux-4.1.6/arc tsc = __native_read_tsc(); canary += tsc + (tsc << 32UL); -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/thread_info.h linux-4.1.6/arch/x86/include/asm/thread_info.h ---- linux-4.1.6.orig/arch/x86/include/asm/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/thread_info.h 2015-09-08 23:49:05.514154783 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/thread_info.h linux-4.1.10/arch/x86/include/asm/thread_info.h +--- linux-4.1.10.orig/arch/x86/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 @@ -55,6 +55,8 @@ __u32 status; /* thread synchronous flags */ __u32 cpu; /* current CPU */ @@ -3277,9 +3299,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/thread_info.h linux-4.1.6/arch/x #define STACK_WARN (THREAD_SIZE/8) /* -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.6/arch/x86/include/asm/uaccess_32.h ---- linux-4.1.6.orig/arch/x86/include/asm/uaccess_32.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/uaccess_32.h 2015-09-08 23:49:05.514154783 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.10/arch/x86/include/asm/uaccess_32.h +--- linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uaccess_32.h 2015-10-07 18:00:07.000000000 +0200 @@ -70,7 +70,8 @@ * @from: Source address, in kernel space. * @n: Number of bytes to copy. @@ -3300,9 +3322,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.6/arch/x8 * * Copy data from user space to kernel space. Caller must check * the specified block with access_ok() before calling this function. -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uaccess.h linux-4.1.6/arch/x86/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/x86/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/uaccess.h 2015-09-08 23:49:05.514154783 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess.h linux-4.1.10/arch/x86/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/x86/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -74,7 +74,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -3353,9 +3375,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uaccess.h linux-4.1.6/arch/x86/i * * This macro copies a single simple value from kernel space to user * space. It supports simple types like char and int, but not larger -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.6/arch/x86/include/asm/uv/uv_bau.h ---- linux-4.1.6.orig/arch/x86/include/asm/uv/uv_bau.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/uv/uv_bau.h 2015-09-08 23:49:05.514154783 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h +--- linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h 2015-10-07 18:00:07.000000000 +0200 @@ -615,9 +615,9 @@ cycles_t send_message; cycles_t period_end; @@ -3389,9 +3411,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.6/arch/x86 return 1; } -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.6/arch/x86/include/asm/uv/uv_hub.h ---- linux-4.1.6.orig/arch/x86/include/asm/uv/uv_hub.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/uv/uv_hub.h 2015-09-08 23:49:05.518154339 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h +--- linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h 2015-10-07 18:00:07.000000000 +0200 @@ -492,7 +492,7 @@ unsigned short nr_online_cpus; unsigned short pnode; @@ -3401,9 +3423,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.6/arch/x86 unsigned long nmi_count; /* obsolete, see uv_hub_nmi */ }; extern struct uv_blade_info *uv_blade_info; -diff -Nur linux-4.1.6.orig/arch/x86/Kconfig linux-4.1.6/arch/x86/Kconfig ---- linux-4.1.6.orig/arch/x86/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/Kconfig 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/Kconfig linux-4.1.10/arch/x86/Kconfig +--- linux-4.1.10.orig/arch/x86/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -22,6 +22,7 @@ ### Arch settings config X86 @@ -3434,9 +3456,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/Kconfig linux-4.1.6/arch/x86/Kconfig ---help--- Enable maximum number of CPUS and NUMA Nodes for this architecture. If unsure, say N. -diff -Nur linux-4.1.6.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.6/arch/x86/kernel/apic/io_apic.c ---- linux-4.1.6.orig/arch/x86/kernel/apic/io_apic.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/apic/io_apic.c 2015-09-08 23:49:05.518154339 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.10/arch/x86/kernel/apic/io_apic.c +--- linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/apic/io_apic.c 2015-10-07 18:00:07.000000000 +0200 @@ -1891,7 +1891,8 @@ static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg) { @@ -3447,9 +3469,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.6/arch/x86/k mask_ioapic(cfg); return true; } -diff -Nur linux-4.1.6.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.6/arch/x86/kernel/apic/x2apic_uv_x.c ---- linux-4.1.6.orig/arch/x86/kernel/apic/x2apic_uv_x.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/apic/x2apic_uv_x.c 2015-09-08 23:49:05.518154339 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c +--- linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-07 18:00:07.000000000 +0200 @@ -949,7 +949,7 @@ uv_blade_info[blade].pnode = pnode; uv_blade_info[blade].nr_possible_cpus = 0; @@ -3459,9 +3481,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.6/arch/x min_pnode = min(pnode, min_pnode); max_pnode = max(pnode, max_pnode); blade++; -diff -Nur linux-4.1.6.orig/arch/x86/kernel/asm-offsets.c linux-4.1.6/arch/x86/kernel/asm-offsets.c ---- linux-4.1.6.orig/arch/x86/kernel/asm-offsets.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/asm-offsets.c 2015-09-08 23:49:05.522153895 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c linux-4.1.10/arch/x86/kernel/asm-offsets.c +--- linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 @@ -32,6 +32,7 @@ OFFSET(TI_flags, thread_info, flags); OFFSET(TI_status, thread_info, status); @@ -3476,9 +3498,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/asm-offsets.c linux-4.1.6/arch/x86/ke DEFINE(PTREGS_SIZE, sizeof(struct pt_regs)); + DEFINE(_PREEMPT_ENABLED, PREEMPT_ENABLED); } -diff -Nur linux-4.1.6.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.6/arch/x86/kernel/cpu/mcheck/mce.c ---- linux-4.1.6.orig/arch/x86/kernel/cpu/mcheck/mce.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/cpu/mcheck/mce.c 2015-09-08 23:49:05.522153895 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c +--- linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-07 18:00:07.000000000 +0200 @@ -41,6 +41,8 @@ #include <linux/debugfs.h> #include <linux/irq_work.h> @@ -3720,9 +3742,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.6/arch/x86 if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) { err = -ENOMEM; goto err_out; -diff -Nur linux-4.1.6.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.6/arch/x86/kernel/dumpstack_32.c ---- linux-4.1.6.orig/arch/x86/kernel/dumpstack_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/dumpstack_32.c 2015-09-08 23:49:05.526153452 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.10/arch/x86/kernel/dumpstack_32.c +--- linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/dumpstack_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -42,7 +42,7 @@ unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data) @@ -3741,9 +3763,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.6/arch/x86/k } EXPORT_SYMBOL(dump_trace); -diff -Nur linux-4.1.6.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.6/arch/x86/kernel/dumpstack_64.c ---- linux-4.1.6.orig/arch/x86/kernel/dumpstack_64.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/dumpstack_64.c 2015-09-08 23:49:05.526153452 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.10/arch/x86/kernel/dumpstack_64.c +--- linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/dumpstack_64.c 2015-10-07 18:00:07.000000000 +0200 @@ -152,7 +152,7 @@ unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data) @@ -3780,9 +3802,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.6/arch/x86/k pr_cont("\n"); show_trace_log_lvl(task, regs, sp, bp, log_lvl); -diff -Nur linux-4.1.6.orig/arch/x86/kernel/entry_32.S linux-4.1.6/arch/x86/kernel/entry_32.S ---- linux-4.1.6.orig/arch/x86/kernel/entry_32.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/entry_32.S 2015-09-08 23:49:05.526153452 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_32.S linux-4.1.10/arch/x86/kernel/entry_32.S +--- linux-4.1.10.orig/arch/x86/kernel/entry_32.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/entry_32.S 2015-10-07 18:00:07.000000000 +0200 @@ -359,8 +359,24 @@ ENTRY(resume_kernel) DISABLE_INTERRUPTS(CLBR_ANY) @@ -3826,9 +3848,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/entry_32.S linux-4.1.6/arch/x86/kerne jnz work_resched work_notifysig: # deal with pending signals and -diff -Nur linux-4.1.6.orig/arch/x86/kernel/entry_64.S linux-4.1.6/arch/x86/kernel/entry_64.S ---- linux-4.1.6.orig/arch/x86/kernel/entry_64.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/entry_64.S 2015-09-08 23:49:05.530153010 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_64.S linux-4.1.10/arch/x86/kernel/entry_64.S +--- linux-4.1.10.orig/arch/x86/kernel/entry_64.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/entry_64.S 2015-10-07 18:00:07.000000000 +0200 @@ -370,8 +370,8 @@ /* First do a reschedule test. */ /* edx: work, edi: workmask */ @@ -3891,9 +3913,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/entry_64.S linux-4.1.6/arch/x86/kerne #ifdef CONFIG_XEN idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0 -diff -Nur linux-4.1.6.orig/arch/x86/kernel/irq_32.c linux-4.1.6/arch/x86/kernel/irq_32.c ---- linux-4.1.6.orig/arch/x86/kernel/irq_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/irq_32.c 2015-09-08 23:49:05.530153010 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/irq_32.c linux-4.1.10/arch/x86/kernel/irq_32.c +--- linux-4.1.10.orig/arch/x86/kernel/irq_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/irq_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -135,6 +135,7 @@ cpu, per_cpu(hardirq_stack, cpu), per_cpu(softirq_stack, cpu)); } @@ -3910,9 +3932,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/irq_32.c linux-4.1.6/arch/x86/kernel/ bool handle_irq(unsigned irq, struct pt_regs *regs) { -diff -Nur linux-4.1.6.orig/arch/x86/kernel/process_32.c linux-4.1.6/arch/x86/kernel/process_32.c ---- linux-4.1.6.orig/arch/x86/kernel/process_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/process_32.c 2015-09-08 23:49:05.530153010 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/process_32.c linux-4.1.10/arch/x86/kernel/process_32.c +--- linux-4.1.10.orig/arch/x86/kernel/process_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/process_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -35,6 +35,7 @@ #include <linux/uaccess.h> #include <linux/io.h> @@ -3966,10 +3988,10 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/process_32.c linux-4.1.6/arch/x86/ker /* * Leave lazy mode, flushing any hypercalls made here. * This must be done before restoring TLS segments so -diff -Nur linux-4.1.6.orig/arch/x86/kernel/signal.c linux-4.1.6/arch/x86/kernel/signal.c ---- linux-4.1.6.orig/arch/x86/kernel/signal.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/signal.c 2015-09-08 23:49:05.534152567 +0200 -@@ -727,6 +727,14 @@ +diff -Nur linux-4.1.10.orig/arch/x86/kernel/signal.c linux-4.1.10/arch/x86/kernel/signal.c +--- linux-4.1.10.orig/arch/x86/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/signal.c 2015-10-07 18:00:07.000000000 +0200 +@@ -723,6 +723,14 @@ { user_exit(); @@ -3984,9 +4006,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/signal.c linux-4.1.6/arch/x86/kernel/ if (thread_info_flags & _TIF_UPROBE) uprobe_notify_resume(regs); -diff -Nur linux-4.1.6.orig/arch/x86/kernel/traps.c linux-4.1.6/arch/x86/kernel/traps.c ---- linux-4.1.6.orig/arch/x86/kernel/traps.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/traps.c 2015-09-08 23:49:05.534152567 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/traps.c linux-4.1.10/arch/x86/kernel/traps.c +--- linux-4.1.10.orig/arch/x86/kernel/traps.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/traps.c 2015-10-07 18:00:07.000000000 +0200 @@ -88,9 +88,21 @@ local_irq_enable(); } @@ -4061,9 +4083,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/traps.c linux-4.1.6/arch/x86/kernel/t debug_stack_usage_dec(); exit: -diff -Nur linux-4.1.6.orig/arch/x86/kvm/lapic.c linux-4.1.6/arch/x86/kvm/lapic.c ---- linux-4.1.6.orig/arch/x86/kvm/lapic.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kvm/lapic.c 2015-09-08 23:49:05.534152567 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kvm/lapic.c linux-4.1.10/arch/x86/kvm/lapic.c +--- linux-4.1.10.orig/arch/x86/kvm/lapic.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kvm/lapic.c 2015-10-07 18:00:07.000000000 +0200 @@ -1104,7 +1104,7 @@ static void apic_timer_expired(struct kvm_lapic *apic) { @@ -4164,9 +4186,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kvm/lapic.c linux-4.1.6/arch/x86/kvm/lapic.c } /* -diff -Nur linux-4.1.6.orig/arch/x86/kvm/x86.c linux-4.1.6/arch/x86/kvm/x86.c ---- linux-4.1.6.orig/arch/x86/kvm/x86.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kvm/x86.c 2015-09-08 23:49:05.538152123 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kvm/x86.c linux-4.1.10/arch/x86/kvm/x86.c +--- linux-4.1.10.orig/arch/x86/kvm/x86.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kvm/x86.c 2015-10-07 18:00:07.000000000 +0200 @@ -5813,6 +5813,13 @@ goto out; } @@ -4181,9 +4203,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kvm/x86.c linux-4.1.6/arch/x86/kvm/x86.c r = kvm_mmu_module_init(); if (r) goto out_free_percpu; -diff -Nur linux-4.1.6.orig/arch/x86/lib/usercopy_32.c linux-4.1.6/arch/x86/lib/usercopy_32.c ---- linux-4.1.6.orig/arch/x86/lib/usercopy_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/lib/usercopy_32.c 2015-09-08 23:49:05.754128173 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/lib/usercopy_32.c linux-4.1.10/arch/x86/lib/usercopy_32.c +--- linux-4.1.10.orig/arch/x86/lib/usercopy_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/lib/usercopy_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -647,7 +647,8 @@ * @from: Source address, in kernel space. * @n: Number of bytes to copy. @@ -4204,9 +4226,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/lib/usercopy_32.c linux-4.1.6/arch/x86/lib/u * * Copy data from user space to kernel space. * -diff -Nur linux-4.1.6.orig/arch/x86/mm/fault.c linux-4.1.6/arch/x86/mm/fault.c ---- linux-4.1.6.orig/arch/x86/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/mm/fault.c 2015-09-08 23:49:05.754128173 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/mm/fault.c linux-4.1.10/arch/x86/mm/fault.c +--- linux-4.1.10.orig/arch/x86/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -13,6 +13,7 @@ #include <linux/hugetlb.h> /* hstate_index_to_shift */ #include <linux/prefetch.h> /* prefetchw */ @@ -4227,9 +4249,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/mm/fault.c linux-4.1.6/arch/x86/mm/fault.c bad_area_nosemaphore(regs, error_code, address); return; } -diff -Nur linux-4.1.6.orig/arch/x86/mm/highmem_32.c linux-4.1.6/arch/x86/mm/highmem_32.c ---- linux-4.1.6.orig/arch/x86/mm/highmem_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/mm/highmem_32.c 2015-09-08 23:49:05.754128173 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/mm/highmem_32.c linux-4.1.10/arch/x86/mm/highmem_32.c +--- linux-4.1.10.orig/arch/x86/mm/highmem_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/mm/highmem_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -32,10 +32,11 @@ */ void *kmap_atomic_prot(struct page *page, pgprot_t prot) @@ -4273,9 +4295,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/mm/highmem_32.c linux-4.1.6/arch/x86/mm/high } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/x86/mm/iomap_32.c linux-4.1.6/arch/x86/mm/iomap_32.c ---- linux-4.1.6.orig/arch/x86/mm/iomap_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/mm/iomap_32.c 2015-09-08 23:49:05.754128173 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/mm/iomap_32.c linux-4.1.10/arch/x86/mm/iomap_32.c +--- linux-4.1.10.orig/arch/x86/mm/iomap_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/mm/iomap_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -56,15 +56,22 @@ void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot) @@ -4315,9 +4337,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/mm/iomap_32.c linux-4.1.6/arch/x86/mm/iomap_ + preempt_enable(); } EXPORT_SYMBOL_GPL(iounmap_atomic); -diff -Nur linux-4.1.6.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.6/arch/x86/platform/uv/tlb_uv.c ---- linux-4.1.6.orig/arch/x86/platform/uv/tlb_uv.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/platform/uv/tlb_uv.c 2015-09-08 23:49:05.758127730 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.10/arch/x86/platform/uv/tlb_uv.c +--- linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/platform/uv/tlb_uv.c 2015-10-07 18:00:07.000000000 +0200 @@ -714,9 +714,9 @@ quiesce_local_uvhub(hmaster); @@ -4404,9 +4426,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.6/arch/x86/pl } } -diff -Nur linux-4.1.6.orig/arch/x86/platform/uv/uv_time.c linux-4.1.6/arch/x86/platform/uv/uv_time.c ---- linux-4.1.6.orig/arch/x86/platform/uv/uv_time.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/platform/uv/uv_time.c 2015-09-08 23:49:05.758127730 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c linux-4.1.10/arch/x86/platform/uv/uv_time.c +--- linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/platform/uv/uv_time.c 2015-10-07 18:00:07.000000000 +0200 @@ -58,7 +58,7 @@ /* There is one of these allocated per node */ @@ -4487,9 +4509,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/platform/uv/uv_time.c linux-4.1.6/arch/x86/p } /* -diff -Nur linux-4.1.6.orig/arch/xtensa/mm/fault.c linux-4.1.6/arch/xtensa/mm/fault.c ---- linux-4.1.6.orig/arch/xtensa/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/xtensa/mm/fault.c 2015-09-08 23:49:05.758127730 +0200 +diff -Nur linux-4.1.10.orig/arch/xtensa/mm/fault.c linux-4.1.10/arch/xtensa/mm/fault.c +--- linux-4.1.10.orig/arch/xtensa/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/xtensa/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -15,10 +15,10 @@ #include <linux/mm.h> #include <linux/module.h> @@ -4511,9 +4533,9 @@ diff -Nur linux-4.1.6.orig/arch/xtensa/mm/fault.c linux-4.1.6/arch/xtensa/mm/fau bad_page_fault(regs, address, SIGSEGV); return; } -diff -Nur linux-4.1.6.orig/arch/xtensa/mm/highmem.c linux-4.1.6/arch/xtensa/mm/highmem.c ---- linux-4.1.6.orig/arch/xtensa/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/xtensa/mm/highmem.c 2015-09-08 23:49:05.758127730 +0200 +diff -Nur linux-4.1.10.orig/arch/xtensa/mm/highmem.c linux-4.1.10/arch/xtensa/mm/highmem.c +--- linux-4.1.10.orig/arch/xtensa/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/xtensa/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -42,6 +42,7 @@ enum fixed_addresses idx; unsigned long vaddr; @@ -4530,9 +4552,9 @@ diff -Nur linux-4.1.6.orig/arch/xtensa/mm/highmem.c linux-4.1.6/arch/xtensa/mm/h } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/block/blk-core.c linux-4.1.6/block/blk-core.c ---- linux-4.1.6.orig/block/blk-core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-core.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-core.c linux-4.1.10/block/blk-core.c +--- linux-4.1.10.orig/block/blk-core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-core.c 2015-10-07 18:00:07.000000000 +0200 @@ -100,6 +100,9 @@ INIT_LIST_HEAD(&rq->queuelist); @@ -4608,9 +4630,9 @@ diff -Nur linux-4.1.6.orig/block/blk-core.c linux-4.1.6/block/blk-core.c } void blk_finish_plug(struct blk_plug *plug) -diff -Nur linux-4.1.6.orig/block/blk-ioc.c linux-4.1.6/block/blk-ioc.c ---- linux-4.1.6.orig/block/blk-ioc.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-ioc.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-ioc.c linux-4.1.10/block/blk-ioc.c +--- linux-4.1.10.orig/block/blk-ioc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-ioc.c 2015-10-07 18:00:07.000000000 +0200 @@ -7,6 +7,7 @@ #include <linux/bio.h> #include <linux/blkdev.h> @@ -4637,9 +4659,9 @@ diff -Nur linux-4.1.6.orig/block/blk-ioc.c linux-4.1.6/block/blk-ioc.c goto retry; } } -diff -Nur linux-4.1.6.orig/block/blk-iopoll.c linux-4.1.6/block/blk-iopoll.c ---- linux-4.1.6.orig/block/blk-iopoll.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-iopoll.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-iopoll.c linux-4.1.10/block/blk-iopoll.c +--- linux-4.1.10.orig/block/blk-iopoll.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-iopoll.c 2015-10-07 18:00:07.000000000 +0200 @@ -35,6 +35,7 @@ list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); @@ -4664,9 +4686,9 @@ diff -Nur linux-4.1.6.orig/block/blk-iopoll.c linux-4.1.6/block/blk-iopoll.c } return NOTIFY_OK; -diff -Nur linux-4.1.6.orig/block/blk-mq.c linux-4.1.6/block/blk-mq.c ---- linux-4.1.6.orig/block/blk-mq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-mq.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-mq.c linux-4.1.10/block/blk-mq.c +--- linux-4.1.10.orig/block/blk-mq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-mq.c 2015-10-07 18:00:07.000000000 +0200 @@ -88,7 +88,7 @@ if (!(gfp & __GFP_WAIT)) return -EBUSY; @@ -4802,9 +4824,9 @@ diff -Nur linux-4.1.6.orig/block/blk-mq.c linux-4.1.6/block/blk-mq.c return blk_mq_hctx_cpu_offline(hctx, cpu); /* -diff -Nur linux-4.1.6.orig/block/blk-mq-cpu.c linux-4.1.6/block/blk-mq-cpu.c ---- linux-4.1.6.orig/block/blk-mq-cpu.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-mq-cpu.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-mq-cpu.c linux-4.1.10/block/blk-mq-cpu.c +--- linux-4.1.10.orig/block/blk-mq-cpu.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-mq-cpu.c 2015-10-07 18:00:07.000000000 +0200 @@ -16,7 +16,7 @@ #include "blk-mq.h" @@ -4856,9 +4878,9 @@ diff -Nur linux-4.1.6.orig/block/blk-mq-cpu.c linux-4.1.6/block/blk-mq-cpu.c } void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier, -diff -Nur linux-4.1.6.orig/block/blk-mq.h linux-4.1.6/block/blk-mq.h ---- linux-4.1.6.orig/block/blk-mq.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-mq.h 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-mq.h linux-4.1.10/block/blk-mq.h +--- linux-4.1.10.orig/block/blk-mq.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-mq.h 2015-10-07 18:00:07.000000000 +0200 @@ -76,7 +76,10 @@ static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q, unsigned int cpu) @@ -4886,9 +4908,9 @@ diff -Nur linux-4.1.6.orig/block/blk-mq.h linux-4.1.6/block/blk-mq.h } struct blk_mq_alloc_data { -diff -Nur linux-4.1.6.orig/block/blk-softirq.c linux-4.1.6/block/blk-softirq.c ---- linux-4.1.6.orig/block/blk-softirq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-softirq.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-softirq.c linux-4.1.10/block/blk-softirq.c +--- linux-4.1.10.orig/block/blk-softirq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-softirq.c 2015-10-07 18:00:07.000000000 +0200 @@ -51,6 +51,7 @@ raise_softirq_irqoff(BLOCK_SOFTIRQ); @@ -4913,9 +4935,9 @@ diff -Nur linux-4.1.6.orig/block/blk-softirq.c linux-4.1.6/block/blk-softirq.c } /** -diff -Nur linux-4.1.6.orig/block/bounce.c linux-4.1.6/block/bounce.c ---- linux-4.1.6.orig/block/bounce.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/bounce.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/bounce.c linux-4.1.10/block/bounce.c +--- linux-4.1.10.orig/block/bounce.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/bounce.c 2015-10-07 18:00:07.000000000 +0200 @@ -54,11 +54,11 @@ unsigned long flags; unsigned char *vto; @@ -4930,9 +4952,9 @@ diff -Nur linux-4.1.6.orig/block/bounce.c linux-4.1.6/block/bounce.c } #else /* CONFIG_HIGHMEM */ -diff -Nur linux-4.1.6.orig/crypto/algapi.c linux-4.1.6/crypto/algapi.c ---- linux-4.1.6.orig/crypto/algapi.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/crypto/algapi.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/crypto/algapi.c linux-4.1.10/crypto/algapi.c +--- linux-4.1.10.orig/crypto/algapi.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/crypto/algapi.c 2015-10-07 18:00:07.000000000 +0200 @@ -695,13 +695,13 @@ int crypto_register_notifier(struct notifier_block *nb) @@ -4949,9 +4971,9 @@ diff -Nur linux-4.1.6.orig/crypto/algapi.c linux-4.1.6/crypto/algapi.c } EXPORT_SYMBOL_GPL(crypto_unregister_notifier); -diff -Nur linux-4.1.6.orig/crypto/api.c linux-4.1.6/crypto/api.c ---- linux-4.1.6.orig/crypto/api.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/crypto/api.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/crypto/api.c linux-4.1.10/crypto/api.c +--- linux-4.1.10.orig/crypto/api.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/crypto/api.c 2015-10-07 18:00:07.000000000 +0200 @@ -31,7 +31,7 @@ DECLARE_RWSEM(crypto_alg_sem); EXPORT_SYMBOL_GPL(crypto_alg_sem); @@ -4974,9 +4996,9 @@ diff -Nur linux-4.1.6.orig/crypto/api.c linux-4.1.6/crypto/api.c } return ok; -diff -Nur linux-4.1.6.orig/crypto/internal.h linux-4.1.6/crypto/internal.h ---- linux-4.1.6.orig/crypto/internal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/crypto/internal.h 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/crypto/internal.h linux-4.1.10/crypto/internal.h +--- linux-4.1.10.orig/crypto/internal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/crypto/internal.h 2015-10-07 18:00:07.000000000 +0200 @@ -48,7 +48,7 @@ extern struct list_head crypto_alg_list; @@ -4995,9 +5017,9 @@ diff -Nur linux-4.1.6.orig/crypto/internal.h linux-4.1.6/crypto/internal.h } #endif /* _CRYPTO_INTERNAL_H */ -diff -Nur linux-4.1.6.orig/Documentation/hwlat_detector.txt linux-4.1.6/Documentation/hwlat_detector.txt ---- linux-4.1.6.orig/Documentation/hwlat_detector.txt 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/Documentation/hwlat_detector.txt 2015-09-08 23:49:03.494378758 +0200 +diff -Nur linux-4.1.10.orig/Documentation/hwlat_detector.txt linux-4.1.10/Documentation/hwlat_detector.txt +--- linux-4.1.10.orig/Documentation/hwlat_detector.txt 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/Documentation/hwlat_detector.txt 2015-10-07 18:00:07.000000000 +0200 @@ -0,0 +1,64 @@ +Introduction: +------------- @@ -5063,9 +5085,9 @@ diff -Nur linux-4.1.6.orig/Documentation/hwlat_detector.txt linux-4.1.6/Document +observe any latencies that exceed the threshold (initially 100 usecs), +then we write to a global sample ring buffer of 8K samples, which is +consumed by reading from the "sample" (pipe) debugfs file interface. -diff -Nur linux-4.1.6.orig/Documentation/sysrq.txt linux-4.1.6/Documentation/sysrq.txt ---- linux-4.1.6.orig/Documentation/sysrq.txt 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/Documentation/sysrq.txt 2015-09-08 23:49:03.494378758 +0200 +diff -Nur linux-4.1.10.orig/Documentation/sysrq.txt linux-4.1.10/Documentation/sysrq.txt +--- linux-4.1.10.orig/Documentation/sysrq.txt 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/Documentation/sysrq.txt 2015-10-07 18:00:07.000000000 +0200 @@ -59,10 +59,17 @@ On other - If you know of the key combos for other architectures, please let me know so I can add them to this section. @@ -5086,9 +5108,9 @@ diff -Nur linux-4.1.6.orig/Documentation/sysrq.txt linux-4.1.6/Documentation/sys * What are the 'command' keys? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 'b' - Will immediately reboot the system without syncing or unmounting -diff -Nur linux-4.1.6.orig/Documentation/trace/histograms.txt linux-4.1.6/Documentation/trace/histograms.txt ---- linux-4.1.6.orig/Documentation/trace/histograms.txt 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/Documentation/trace/histograms.txt 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/Documentation/trace/histograms.txt linux-4.1.10/Documentation/trace/histograms.txt +--- linux-4.1.10.orig/Documentation/trace/histograms.txt 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/Documentation/trace/histograms.txt 2015-10-07 18:00:07.000000000 +0200 @@ -0,0 +1,186 @@ + Using the Linux Kernel Latency Histograms + @@ -5276,9 +5298,9 @@ diff -Nur linux-4.1.6.orig/Documentation/trace/histograms.txt linux-4.1.6/Docume +is provided. + +These data are also reset when the wakeup histogram is reset. -diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/acglobal.h linux-4.1.6/drivers/acpi/acpica/acglobal.h ---- linux-4.1.6.orig/drivers/acpi/acpica/acglobal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/acpi/acpica/acglobal.h 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h linux-4.1.10/drivers/acpi/acpica/acglobal.h +--- linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/acglobal.h 2015-10-07 18:00:07.000000000 +0200 @@ -112,7 +112,7 @@ * interrupt level */ @@ -5288,9 +5310,9 @@ diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/acglobal.h linux-4.1.6/drivers/ac ACPI_GLOBAL(acpi_spinlock, acpi_gbl_reference_count_lock); /* Mutex for _OSI support */ -diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/hwregs.c linux-4.1.6/drivers/acpi/acpica/hwregs.c ---- linux-4.1.6.orig/drivers/acpi/acpica/hwregs.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/acpi/acpica/hwregs.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c linux-4.1.10/drivers/acpi/acpica/hwregs.c +--- linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/hwregs.c 2015-10-07 18:00:07.000000000 +0200 @@ -269,14 +269,14 @@ ACPI_BITMASK_ALL_FIXED_STATUS, ACPI_FORMAT_UINT64(acpi_gbl_xpm1a_status.address))); @@ -5308,9 +5330,9 @@ diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/hwregs.c linux-4.1.6/drivers/acpi if (ACPI_FAILURE(status)) { goto exit; -diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/hwxface.c linux-4.1.6/drivers/acpi/acpica/hwxface.c ---- linux-4.1.6.orig/drivers/acpi/acpica/hwxface.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/acpi/acpica/hwxface.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c linux-4.1.10/drivers/acpi/acpica/hwxface.c +--- linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/hwxface.c 2015-10-07 18:00:07.000000000 +0200 @@ -374,7 +374,7 @@ return_ACPI_STATUS(AE_BAD_PARAMETER); } @@ -5329,9 +5351,9 @@ diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/hwxface.c linux-4.1.6/drivers/acp return_ACPI_STATUS(status); } -diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/utmutex.c linux-4.1.6/drivers/acpi/acpica/utmutex.c ---- linux-4.1.6.orig/drivers/acpi/acpica/utmutex.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/acpi/acpica/utmutex.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c linux-4.1.10/drivers/acpi/acpica/utmutex.c +--- linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/utmutex.c 2015-10-07 18:00:07.000000000 +0200 @@ -88,7 +88,7 @@ return_ACPI_STATUS (status); } @@ -5350,9 +5372,9 @@ diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/utmutex.c linux-4.1.6/drivers/acp acpi_os_delete_lock(acpi_gbl_reference_count_lock); /* Delete the reader/writer lock */ -diff -Nur linux-4.1.6.orig/drivers/ata/libata-sff.c linux-4.1.6/drivers/ata/libata-sff.c ---- linux-4.1.6.orig/drivers/ata/libata-sff.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ata/libata-sff.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/ata/libata-sff.c linux-4.1.10/drivers/ata/libata-sff.c +--- linux-4.1.10.orig/drivers/ata/libata-sff.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ata/libata-sff.c 2015-10-07 18:00:07.000000000 +0200 @@ -678,9 +678,9 @@ unsigned long flags; unsigned int consumed; @@ -5401,9 +5423,9 @@ diff -Nur linux-4.1.6.orig/drivers/ata/libata-sff.c linux-4.1.6/drivers/ata/liba } else { buf = page_address(page); consumed = ap->ops->sff_data_xfer(dev, buf + offset, -diff -Nur linux-4.1.6.orig/drivers/char/random.c linux-4.1.6/drivers/char/random.c ---- linux-4.1.6.orig/drivers/char/random.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/char/random.c 2015-09-08 23:49:05.766126841 +0200 +diff -Nur linux-4.1.10.orig/drivers/char/random.c linux-4.1.10/drivers/char/random.c +--- linux-4.1.10.orig/drivers/char/random.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/char/random.c 2015-10-07 18:00:07.000000000 +0200 @@ -776,8 +776,6 @@ } sample; long delta, delta2, delta3; @@ -5455,9 +5477,9 @@ diff -Nur linux-4.1.6.orig/drivers/char/random.c linux-4.1.6/drivers/char/random fast_mix(fast_pool); add_interrupt_bench(cycles); -diff -Nur linux-4.1.6.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.6/drivers/clocksource/tcb_clksrc.c ---- linux-4.1.6.orig/drivers/clocksource/tcb_clksrc.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/clocksource/tcb_clksrc.c 2015-09-08 23:49:05.766126841 +0200 +diff -Nur linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.10/drivers/clocksource/tcb_clksrc.c +--- linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/clocksource/tcb_clksrc.c 2015-10-07 18:00:08.000000000 +0200 @@ -23,8 +23,7 @@ * this 32 bit free-running counter. the second channel is not used. * @@ -5572,9 +5594,9 @@ diff -Nur linux-4.1.6.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.6/drivers/ if (ret) goto err_unregister_clksrc; -diff -Nur linux-4.1.6.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.6/drivers/clocksource/timer-atmel-pit.c ---- linux-4.1.6.orig/drivers/clocksource/timer-atmel-pit.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/clocksource/timer-atmel-pit.c 2015-09-08 23:49:05.766126841 +0200 +diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.10/drivers/clocksource/timer-atmel-pit.c +--- linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/clocksource/timer-atmel-pit.c 2015-10-07 18:00:08.000000000 +0200 @@ -90,6 +90,7 @@ return elapsed; } @@ -5600,9 +5622,9 @@ diff -Nur linux-4.1.6.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.6/dri break; case CLOCK_EVT_MODE_RESUME: break; -diff -Nur linux-4.1.6.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.6/drivers/clocksource/timer-atmel-st.c ---- linux-4.1.6.orig/drivers/clocksource/timer-atmel-st.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/clocksource/timer-atmel-st.c 2015-09-08 23:49:05.766126841 +0200 +diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.10/drivers/clocksource/timer-atmel-st.c +--- linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/clocksource/timer-atmel-st.c 2015-10-07 18:00:08.000000000 +0200 @@ -131,6 +131,7 @@ break; case CLOCK_EVT_MODE_SHUTDOWN: @@ -5611,9 +5633,9 @@ diff -Nur linux-4.1.6.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.6/driv case CLOCK_EVT_MODE_RESUME: irqmask = 0; break; -diff -Nur linux-4.1.6.orig/drivers/cpufreq/cpufreq.c linux-4.1.6/drivers/cpufreq/cpufreq.c ---- linux-4.1.6.orig/drivers/cpufreq/cpufreq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/cpufreq/cpufreq.c 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/cpufreq/cpufreq.c linux-4.1.10/drivers/cpufreq/cpufreq.c +--- linux-4.1.10.orig/drivers/cpufreq/cpufreq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/cpufreq/cpufreq.c 2015-10-07 18:00:08.000000000 +0200 @@ -64,12 +64,6 @@ return cpufreq_driver->target_index || cpufreq_driver->target; } @@ -5750,9 +5772,9 @@ diff -Nur linux-4.1.6.orig/drivers/cpufreq/cpufreq.c linux-4.1.6/drivers/cpufreq return 0; } -diff -Nur linux-4.1.6.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.6/drivers/cpufreq/Kconfig.x86 ---- linux-4.1.6.orig/drivers/cpufreq/Kconfig.x86 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/cpufreq/Kconfig.x86 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.10/drivers/cpufreq/Kconfig.x86 +--- linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/cpufreq/Kconfig.x86 2015-10-07 18:00:08.000000000 +0200 @@ -123,7 +123,7 @@ config X86_POWERNOW_K8 @@ -5762,9 +5784,9 @@ diff -Nur linux-4.1.6.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.6/drivers/cpufr help This adds the CPUFreq driver for K8/early Opteron/Athlon64 processors. Support for K10 and newer processors is now in acpi-cpufreq. -diff -Nur linux-4.1.6.orig/drivers/gpio/gpio-omap.c linux-4.1.6/drivers/gpio/gpio-omap.c ---- linux-4.1.6.orig/drivers/gpio/gpio-omap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/gpio/gpio-omap.c 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/gpio/gpio-omap.c linux-4.1.10/drivers/gpio/gpio-omap.c +--- linux-4.1.10.orig/drivers/gpio/gpio-omap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/gpio/gpio-omap.c 2015-10-07 18:00:08.000000000 +0200 @@ -57,7 +57,7 @@ u32 saved_datain; u32 level_mask; @@ -6045,9 +6067,9 @@ diff -Nur linux-4.1.6.orig/drivers/gpio/gpio-omap.c linux-4.1.6/drivers/gpio/gpi return 0; } -diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1.6/drivers/gpu/drm/i915/i915_gem_execbuffer.c ---- linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c +--- linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-07 18:00:08.000000000 +0200 @@ -32,6 +32,7 @@ #include "i915_trace.h" #include "intel_drv.h" @@ -6075,9 +6097,9 @@ diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1. i915_gem_execbuffer_move_to_active(vmas, ring); i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); -diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.6/drivers/gpu/drm/i915/i915_gem_shrinker.c ---- linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c +--- linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-07 18:00:08.000000000 +0200 @@ -39,7 +39,7 @@ if (!mutex_is_locked(mutex)) return false; @@ -6087,9 +6109,9 @@ diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.6/ return mutex->owner == task; #else /* Since UP may be pre-empted, we cannot assume that we own the lock */ -diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.6/drivers/gpu/drm/i915/intel_display.c ---- linux-4.1.6.orig/drivers/gpu/drm/i915/intel_display.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/gpu/drm/i915/intel_display.c 2015-09-08 23:49:05.822120634 +0200 +diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.10/drivers/gpu/drm/i915/intel_display.c +--- linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/gpu/drm/i915/intel_display.c 2015-10-07 18:00:08.000000000 +0200 @@ -10086,7 +10086,7 @@ struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); @@ -6099,9 +6121,9 @@ diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.6/driv if (crtc == NULL) return; -diff -Nur linux-4.1.6.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.6/drivers/i2c/busses/i2c-omap.c ---- linux-4.1.6.orig/drivers/i2c/busses/i2c-omap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/i2c/busses/i2c-omap.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.10/drivers/i2c/busses/i2c-omap.c +--- linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/i2c/busses/i2c-omap.c 2015-10-07 18:00:08.000000000 +0200 @@ -996,15 +996,12 @@ u16 mask; u16 stat; @@ -6119,9 +6141,9 @@ diff -Nur linux-4.1.6.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.6/drivers/i2c return ret; } -diff -Nur linux-4.1.6.orig/drivers/ide/alim15x3.c linux-4.1.6/drivers/ide/alim15x3.c ---- linux-4.1.6.orig/drivers/ide/alim15x3.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/alim15x3.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/alim15x3.c linux-4.1.10/drivers/ide/alim15x3.c +--- linux-4.1.10.orig/drivers/ide/alim15x3.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/alim15x3.c 2015-10-07 18:00:08.000000000 +0200 @@ -234,7 +234,7 @@ isa_dev = pci_get_device(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, NULL); @@ -6140,9 +6162,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/alim15x3.c linux-4.1.6/drivers/ide/alim15 return 0; } -diff -Nur linux-4.1.6.orig/drivers/ide/hpt366.c linux-4.1.6/drivers/ide/hpt366.c ---- linux-4.1.6.orig/drivers/ide/hpt366.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/hpt366.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/hpt366.c linux-4.1.10/drivers/ide/hpt366.c +--- linux-4.1.10.orig/drivers/ide/hpt366.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/hpt366.c 2015-10-07 18:00:08.000000000 +0200 @@ -1241,7 +1241,7 @@ dma_old = inb(base + 2); @@ -6161,9 +6183,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/hpt366.c linux-4.1.6/drivers/ide/hpt366.c printk(KERN_INFO " %s: BM-DMA at 0x%04lx-0x%04lx\n", hwif->name, base, base + 7); -diff -Nur linux-4.1.6.orig/drivers/ide/ide-io.c linux-4.1.6/drivers/ide/ide-io.c ---- linux-4.1.6.orig/drivers/ide/ide-io.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-io.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-io.c linux-4.1.10/drivers/ide/ide-io.c +--- linux-4.1.10.orig/drivers/ide/ide-io.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-io.c 2015-10-07 18:00:08.000000000 +0200 @@ -659,7 +659,7 @@ /* disable_irq_nosync ?? */ disable_irq(hwif->irq); @@ -6173,9 +6195,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-io.c linux-4.1.6/drivers/ide/ide-io.c if (hwif->polling) { startstop = handler(drive); } else if (drive_is_ready(drive)) { -diff -Nur linux-4.1.6.orig/drivers/ide/ide-iops.c linux-4.1.6/drivers/ide/ide-iops.c ---- linux-4.1.6.orig/drivers/ide/ide-iops.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-iops.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-iops.c linux-4.1.10/drivers/ide/ide-iops.c +--- linux-4.1.10.orig/drivers/ide/ide-iops.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-iops.c 2015-10-07 18:00:08.000000000 +0200 @@ -129,12 +129,12 @@ if ((stat & ATA_BUSY) == 0) break; @@ -6191,9 +6213,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-iops.c linux-4.1.6/drivers/ide/ide-io } /* * Allow status to settle, then read it again. -diff -Nur linux-4.1.6.orig/drivers/ide/ide-io-std.c linux-4.1.6/drivers/ide/ide-io-std.c ---- linux-4.1.6.orig/drivers/ide/ide-io-std.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-io-std.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-io-std.c linux-4.1.10/drivers/ide/ide-io-std.c +--- linux-4.1.10.orig/drivers/ide/ide-io-std.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-io-std.c 2015-10-07 18:00:08.000000000 +0200 @@ -175,7 +175,7 @@ unsigned long uninitialized_var(flags); @@ -6230,9 +6252,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-io-std.c linux-4.1.6/drivers/ide/ide- if (((len + 1) & 3) < 2) return; -diff -Nur linux-4.1.6.orig/drivers/ide/ide-probe.c linux-4.1.6/drivers/ide/ide-probe.c ---- linux-4.1.6.orig/drivers/ide/ide-probe.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-probe.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-probe.c linux-4.1.10/drivers/ide/ide-probe.c +--- linux-4.1.10.orig/drivers/ide/ide-probe.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-probe.c 2015-10-07 18:00:08.000000000 +0200 @@ -196,10 +196,10 @@ int bswap = 1; @@ -6246,9 +6268,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-probe.c linux-4.1.6/drivers/ide/ide-p drive->dev_flags |= IDE_DFLAG_ID_READ; #ifdef DEBUG -diff -Nur linux-4.1.6.orig/drivers/ide/ide-taskfile.c linux-4.1.6/drivers/ide/ide-taskfile.c ---- linux-4.1.6.orig/drivers/ide/ide-taskfile.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-taskfile.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-taskfile.c linux-4.1.10/drivers/ide/ide-taskfile.c +--- linux-4.1.10.orig/drivers/ide/ide-taskfile.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-taskfile.c 2015-10-07 18:00:08.000000000 +0200 @@ -250,7 +250,7 @@ page_is_high = PageHighMem(page); @@ -6276,9 +6298,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-taskfile.c linux-4.1.6/drivers/ide/id ide_set_handler(drive, &task_pio_intr, WAIT_WORSTCASE); -diff -Nur linux-4.1.6.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux-4.1.6/drivers/infiniband/ulp/ipoib/ipoib_multicast.c ---- linux-4.1.6.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +--- linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-07 18:00:08.000000000 +0200 @@ -821,7 +821,7 @@ ipoib_dbg_mcast(priv, "restarting multicast task\n"); @@ -6297,9 +6319,9 @@ diff -Nur linux-4.1.6.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux- /* * make sure the in-flight joins have finished before we attempt -diff -Nur linux-4.1.6.orig/drivers/input/gameport/gameport.c linux-4.1.6/drivers/input/gameport/gameport.c ---- linux-4.1.6.orig/drivers/input/gameport/gameport.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/input/gameport/gameport.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/input/gameport/gameport.c linux-4.1.10/drivers/input/gameport/gameport.c +--- linux-4.1.10.orig/drivers/input/gameport/gameport.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/input/gameport/gameport.c 2015-10-07 18:00:08.000000000 +0200 @@ -124,12 +124,12 @@ tx = 1 << 30; @@ -6329,9 +6351,9 @@ diff -Nur linux-4.1.6.orig/drivers/input/gameport/gameport.c linux-4.1.6/drivers udelay(i * 10); if (t2 - t1 < tx) tx = t2 - t1; } -diff -Nur linux-4.1.6.orig/drivers/leds/trigger/Kconfig linux-4.1.6/drivers/leds/trigger/Kconfig ---- linux-4.1.6.orig/drivers/leds/trigger/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/leds/trigger/Kconfig 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/leds/trigger/Kconfig linux-4.1.10/drivers/leds/trigger/Kconfig +--- linux-4.1.10.orig/drivers/leds/trigger/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/leds/trigger/Kconfig 2015-10-07 18:00:08.000000000 +0200 @@ -61,7 +61,7 @@ config LEDS_TRIGGER_CPU @@ -6341,9 +6363,9 @@ diff -Nur linux-4.1.6.orig/drivers/leds/trigger/Kconfig linux-4.1.6/drivers/leds help This allows LEDs to be controlled by active CPUs. This shows the active CPUs across an array of LEDs so you can see which -diff -Nur linux-4.1.6.orig/drivers/md/bcache/Kconfig linux-4.1.6/drivers/md/bcache/Kconfig ---- linux-4.1.6.orig/drivers/md/bcache/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/md/bcache/Kconfig 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/md/bcache/Kconfig linux-4.1.10/drivers/md/bcache/Kconfig +--- linux-4.1.10.orig/drivers/md/bcache/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/md/bcache/Kconfig 2015-10-07 18:00:08.000000000 +0200 @@ -1,6 +1,7 @@ config BCACHE @@ -6352,9 +6374,9 @@ diff -Nur linux-4.1.6.orig/drivers/md/bcache/Kconfig linux-4.1.6/drivers/md/bcac ---help--- Allows a block device to be used as cache for other devices; uses a btree for indexing and the layout is optimized for SSDs. -diff -Nur linux-4.1.6.orig/drivers/md/dm.c linux-4.1.6/drivers/md/dm.c ---- linux-4.1.6.orig/drivers/md/dm.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/md/dm.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/md/dm.c linux-4.1.10/drivers/md/dm.c +--- linux-4.1.10.orig/drivers/md/dm.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/md/dm.c 2015-10-07 18:00:08.000000000 +0200 @@ -2132,7 +2132,7 @@ /* Establish tio->ti before queuing work (map_tio_request) */ tio->ti = ti; @@ -6364,9 +6386,9 @@ diff -Nur linux-4.1.6.orig/drivers/md/dm.c linux-4.1.6/drivers/md/dm.c } goto out; -diff -Nur linux-4.1.6.orig/drivers/md/raid5.c linux-4.1.6/drivers/md/raid5.c ---- linux-4.1.6.orig/drivers/md/raid5.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/md/raid5.c 2015-09-08 23:49:06.078092247 +0200 +diff -Nur linux-4.1.10.orig/drivers/md/raid5.c linux-4.1.10/drivers/md/raid5.c +--- linux-4.1.10.orig/drivers/md/raid5.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/md/raid5.c 2015-10-07 18:00:08.000000000 +0200 @@ -1918,8 +1918,9 @@ struct raid5_percpu *percpu; unsigned long cpu; @@ -6388,7 +6410,7 @@ diff -Nur linux-4.1.6.orig/drivers/md/raid5.c linux-4.1.6/drivers/md/raid5.c } static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp) -@@ -6350,6 +6352,7 @@ +@@ -6361,6 +6363,7 @@ __func__, cpu); break; } @@ -6396,10 +6418,7867 @@ diff -Nur linux-4.1.6.orig/drivers/md/raid5.c linux-4.1.6/drivers/md/raid5.c } put_online_cpus(); -diff -Nur linux-4.1.6.orig/drivers/md/raid5.h linux-4.1.6/drivers/md/raid5.h ---- linux-4.1.6.orig/drivers/md/raid5.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/md/raid5.h 2015-09-08 23:49:06.078092247 +0200 -@@ -494,6 +494,7 @@ +diff -Nur linux-4.1.10.orig/drivers/md/raid5.c.orig linux-4.1.10/drivers/md/raid5.c.orig +--- linux-4.1.10.orig/drivers/md/raid5.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/drivers/md/raid5.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,7853 @@ ++/* ++ * raid5.c : Multiple Devices driver for Linux ++ * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman ++ * Copyright (C) 1999, 2000 Ingo Molnar ++ * Copyright (C) 2002, 2003 H. Peter Anvin ++ * ++ * RAID-4/5/6 management functions. ++ * Thanks to Penguin Computing for making the RAID-6 development possible ++ * by donating a test server! ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2, or (at your option) ++ * any later version. ++ * ++ * You should have received a copy of the GNU General Public License ++ * (for example /usr/src/linux/COPYING); if not, write to the Free ++ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ++ */ ++ ++/* ++ * BITMAP UNPLUGGING: ++ * ++ * The sequencing for updating the bitmap reliably is a little ++ * subtle (and I got it wrong the first time) so it deserves some ++ * explanation. ++ * ++ * We group bitmap updates into batches. Each batch has a number. ++ * We may write out several batches at once, but that isn't very important. ++ * conf->seq_write is the number of the last batch successfully written. ++ * conf->seq_flush is the number of the last batch that was closed to ++ * new additions. ++ * When we discover that we will need to write to any block in a stripe ++ * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq ++ * the number of the batch it will be in. This is seq_flush+1. ++ * When we are ready to do a write, if that batch hasn't been written yet, ++ * we plug the array and queue the stripe for later. ++ * When an unplug happens, we increment bm_flush, thus closing the current ++ * batch. ++ * When we notice that bm_flush > bm_write, we write out all pending updates ++ * to the bitmap, and advance bm_write to where bm_flush was. ++ * This may occasionally write a bit out twice, but is sure never to ++ * miss any bits. ++ */ ++ ++#include <linux/blkdev.h> ++#include <linux/kthread.h> ++#include <linux/raid/pq.h> ++#include <linux/async_tx.h> ++#include <linux/module.h> ++#include <linux/async.h> ++#include <linux/seq_file.h> ++#include <linux/cpu.h> ++#include <linux/slab.h> ++#include <linux/ratelimit.h> ++#include <linux/nodemask.h> ++#include <linux/flex_array.h> ++#include <trace/events/block.h> ++ ++#include "md.h" ++#include "raid5.h" ++#include "raid0.h" ++#include "bitmap.h" ++ ++#define cpu_to_group(cpu) cpu_to_node(cpu) ++#define ANY_GROUP NUMA_NO_NODE ++ ++static bool devices_handle_discard_safely = false; ++module_param(devices_handle_discard_safely, bool, 0644); ++MODULE_PARM_DESC(devices_handle_discard_safely, ++ "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions"); ++static struct workqueue_struct *raid5_wq; ++/* ++ * Stripe cache ++ */ ++ ++#define NR_STRIPES 256 ++#define STRIPE_SIZE PAGE_SIZE ++#define STRIPE_SHIFT (PAGE_SHIFT - 9) ++#define STRIPE_SECTORS (STRIPE_SIZE>>9) ++#define IO_THRESHOLD 1 ++#define BYPASS_THRESHOLD 1 ++#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) ++#define HASH_MASK (NR_HASH - 1) ++#define MAX_STRIPE_BATCH 8 ++ ++static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) ++{ ++ int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; ++ return &conf->stripe_hashtbl[hash]; ++} ++ ++static inline int stripe_hash_locks_hash(sector_t sect) ++{ ++ return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK; ++} ++ ++static inline void lock_device_hash_lock(struct r5conf *conf, int hash) ++{ ++ spin_lock_irq(conf->hash_locks + hash); ++ spin_lock(&conf->device_lock); ++} ++ ++static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) ++{ ++ spin_unlock(&conf->device_lock); ++ spin_unlock_irq(conf->hash_locks + hash); ++} ++ ++static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) ++{ ++ int i; ++ local_irq_disable(); ++ spin_lock(conf->hash_locks); ++ for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) ++ spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); ++ spin_lock(&conf->device_lock); ++} ++ ++static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) ++{ ++ int i; ++ spin_unlock(&conf->device_lock); ++ for (i = NR_STRIPE_HASH_LOCKS; i; i--) ++ spin_unlock(conf->hash_locks + i - 1); ++ local_irq_enable(); ++} ++ ++/* bio's attached to a stripe+device for I/O are linked together in bi_sector ++ * order without overlap. There may be several bio's per stripe+device, and ++ * a bio could span several devices. ++ * When walking this list for a particular stripe+device, we must never proceed ++ * beyond a bio that extends past this device, as the next bio might no longer ++ * be valid. ++ * This function is used to determine the 'next' bio in the list, given the sector ++ * of the current stripe+device ++ */ ++static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) ++{ ++ int sectors = bio_sectors(bio); ++ if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS) ++ return bio->bi_next; ++ else ++ return NULL; ++} ++ ++/* ++ * We maintain a biased count of active stripes in the bottom 16 bits of ++ * bi_phys_segments, and a count of processed stripes in the upper 16 bits ++ */ ++static inline int raid5_bi_processed_stripes(struct bio *bio) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ return (atomic_read(segments) >> 16) & 0xffff; ++} ++ ++static inline int raid5_dec_bi_active_stripes(struct bio *bio) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ return atomic_sub_return(1, segments) & 0xffff; ++} ++ ++static inline void raid5_inc_bi_active_stripes(struct bio *bio) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ atomic_inc(segments); ++} ++ ++static inline void raid5_set_bi_processed_stripes(struct bio *bio, ++ unsigned int cnt) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ int old, new; ++ ++ do { ++ old = atomic_read(segments); ++ new = (old & 0xffff) | (cnt << 16); ++ } while (atomic_cmpxchg(segments, old, new) != old); ++} ++ ++static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ atomic_set(segments, cnt); ++} ++ ++/* Find first data disk in a raid6 stripe */ ++static inline int raid6_d0(struct stripe_head *sh) ++{ ++ if (sh->ddf_layout) ++ /* ddf always start from first device */ ++ return 0; ++ /* md starts just after Q block */ ++ if (sh->qd_idx == sh->disks - 1) ++ return 0; ++ else ++ return sh->qd_idx + 1; ++} ++static inline int raid6_next_disk(int disk, int raid_disks) ++{ ++ disk++; ++ return (disk < raid_disks) ? disk : 0; ++} ++ ++/* When walking through the disks in a raid5, starting at raid6_d0, ++ * We need to map each disk to a 'slot', where the data disks are slot ++ * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk ++ * is raid_disks-1. This help does that mapping. ++ */ ++static int raid6_idx_to_slot(int idx, struct stripe_head *sh, ++ int *count, int syndrome_disks) ++{ ++ int slot = *count; ++ ++ if (sh->ddf_layout) ++ (*count)++; ++ if (idx == sh->pd_idx) ++ return syndrome_disks; ++ if (idx == sh->qd_idx) ++ return syndrome_disks + 1; ++ if (!sh->ddf_layout) ++ (*count)++; ++ return slot; ++} ++ ++static void return_io(struct bio *return_bi) ++{ ++ struct bio *bi = return_bi; ++ while (bi) { ++ ++ return_bi = bi->bi_next; ++ bi->bi_next = NULL; ++ bi->bi_iter.bi_size = 0; ++ trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), ++ bi, 0); ++ bio_endio(bi, 0); ++ bi = return_bi; ++ } ++} ++ ++static void print_raid5_conf (struct r5conf *conf); ++ ++static int stripe_operations_active(struct stripe_head *sh) ++{ ++ return sh->check_state || sh->reconstruct_state || ++ test_bit(STRIPE_BIOFILL_RUN, &sh->state) || ++ test_bit(STRIPE_COMPUTE_RUN, &sh->state); ++} ++ ++static void raid5_wakeup_stripe_thread(struct stripe_head *sh) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ struct r5worker_group *group; ++ int thread_cnt; ++ int i, cpu = sh->cpu; ++ ++ if (!cpu_online(cpu)) { ++ cpu = cpumask_any(cpu_online_mask); ++ sh->cpu = cpu; ++ } ++ ++ if (list_empty(&sh->lru)) { ++ struct r5worker_group *group; ++ group = conf->worker_groups + cpu_to_group(cpu); ++ list_add_tail(&sh->lru, &group->handle_list); ++ group->stripes_cnt++; ++ sh->group = group; ++ } ++ ++ if (conf->worker_cnt_per_group == 0) { ++ md_wakeup_thread(conf->mddev->thread); ++ return; ++ } ++ ++ group = conf->worker_groups + cpu_to_group(sh->cpu); ++ ++ group->workers[0].working = true; ++ /* at least one worker should run to avoid race */ ++ queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); ++ ++ thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; ++ /* wakeup more workers */ ++ for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { ++ if (group->workers[i].working == false) { ++ group->workers[i].working = true; ++ queue_work_on(sh->cpu, raid5_wq, ++ &group->workers[i].work); ++ thread_cnt--; ++ } ++ } ++} ++ ++static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, ++ struct list_head *temp_inactive_list) ++{ ++ BUG_ON(!list_empty(&sh->lru)); ++ BUG_ON(atomic_read(&conf->active_stripes)==0); ++ if (test_bit(STRIPE_HANDLE, &sh->state)) { ++ if (test_bit(STRIPE_DELAYED, &sh->state) && ++ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ list_add_tail(&sh->lru, &conf->delayed_list); ++ else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && ++ sh->bm_seq - conf->seq_write > 0) ++ list_add_tail(&sh->lru, &conf->bitmap_list); ++ else { ++ clear_bit(STRIPE_DELAYED, &sh->state); ++ clear_bit(STRIPE_BIT_DELAY, &sh->state); ++ if (conf->worker_cnt_per_group == 0) { ++ list_add_tail(&sh->lru, &conf->handle_list); ++ } else { ++ raid5_wakeup_stripe_thread(sh); ++ return; ++ } ++ } ++ md_wakeup_thread(conf->mddev->thread); ++ } else { ++ BUG_ON(stripe_operations_active(sh)); ++ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ if (atomic_dec_return(&conf->preread_active_stripes) ++ < IO_THRESHOLD) ++ md_wakeup_thread(conf->mddev->thread); ++ atomic_dec(&conf->active_stripes); ++ if (!test_bit(STRIPE_EXPANDING, &sh->state)) ++ list_add_tail(&sh->lru, temp_inactive_list); ++ } ++} ++ ++static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, ++ struct list_head *temp_inactive_list) ++{ ++ if (atomic_dec_and_test(&sh->count)) ++ do_release_stripe(conf, sh, temp_inactive_list); ++} ++ ++/* ++ * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list ++ * ++ * Be careful: Only one task can add/delete stripes from temp_inactive_list at ++ * given time. Adding stripes only takes device lock, while deleting stripes ++ * only takes hash lock. ++ */ ++static void release_inactive_stripe_list(struct r5conf *conf, ++ struct list_head *temp_inactive_list, ++ int hash) ++{ ++ int size; ++ bool do_wakeup = false; ++ unsigned long flags; ++ ++ if (hash == NR_STRIPE_HASH_LOCKS) { ++ size = NR_STRIPE_HASH_LOCKS; ++ hash = NR_STRIPE_HASH_LOCKS - 1; ++ } else ++ size = 1; ++ while (size) { ++ struct list_head *list = &temp_inactive_list[size - 1]; ++ ++ /* ++ * We don't hold any lock here yet, get_active_stripe() might ++ * remove stripes from the list ++ */ ++ if (!list_empty_careful(list)) { ++ spin_lock_irqsave(conf->hash_locks + hash, flags); ++ if (list_empty(conf->inactive_list + hash) && ++ !list_empty(list)) ++ atomic_dec(&conf->empty_inactive_list_nr); ++ list_splice_tail_init(list, conf->inactive_list + hash); ++ do_wakeup = true; ++ spin_unlock_irqrestore(conf->hash_locks + hash, flags); ++ } ++ size--; ++ hash--; ++ } ++ ++ if (do_wakeup) { ++ wake_up(&conf->wait_for_stripe); ++ if (conf->retry_read_aligned) ++ md_wakeup_thread(conf->mddev->thread); ++ } ++} ++ ++/* should hold conf->device_lock already */ ++static int release_stripe_list(struct r5conf *conf, ++ struct list_head *temp_inactive_list) ++{ ++ struct stripe_head *sh; ++ int count = 0; ++ struct llist_node *head; ++ ++ head = llist_del_all(&conf->released_stripes); ++ head = llist_reverse_order(head); ++ while (head) { ++ int hash; ++ ++ sh = llist_entry(head, struct stripe_head, release_list); ++ head = llist_next(head); ++ /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ ++ smp_mb(); ++ clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); ++ /* ++ * Don't worry the bit is set here, because if the bit is set ++ * again, the count is always > 1. This is true for ++ * STRIPE_ON_UNPLUG_LIST bit too. ++ */ ++ hash = sh->hash_lock_index; ++ __release_stripe(conf, sh, &temp_inactive_list[hash]); ++ count++; ++ } ++ ++ return count; ++} ++ ++static void release_stripe(struct stripe_head *sh) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ unsigned long flags; ++ struct list_head list; ++ int hash; ++ bool wakeup; ++ ++ /* Avoid release_list until the last reference. ++ */ ++ if (atomic_add_unless(&sh->count, -1, 1)) ++ return; ++ ++ if (unlikely(!conf->mddev->thread) || ++ test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) ++ goto slow_path; ++ wakeup = llist_add(&sh->release_list, &conf->released_stripes); ++ if (wakeup) ++ md_wakeup_thread(conf->mddev->thread); ++ return; ++slow_path: ++ local_irq_save(flags); ++ /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ ++ if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { ++ INIT_LIST_HEAD(&list); ++ hash = sh->hash_lock_index; ++ do_release_stripe(conf, sh, &list); ++ spin_unlock(&conf->device_lock); ++ release_inactive_stripe_list(conf, &list, hash); ++ } ++ local_irq_restore(flags); ++} ++ ++static inline void remove_hash(struct stripe_head *sh) ++{ ++ pr_debug("remove_hash(), stripe %llu\n", ++ (unsigned long long)sh->sector); ++ ++ hlist_del_init(&sh->hash); ++} ++ ++static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) ++{ ++ struct hlist_head *hp = stripe_hash(conf, sh->sector); ++ ++ pr_debug("insert_hash(), stripe %llu\n", ++ (unsigned long long)sh->sector); ++ ++ hlist_add_head(&sh->hash, hp); ++} ++ ++/* find an idle stripe, make sure it is unhashed, and return it. */ ++static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) ++{ ++ struct stripe_head *sh = NULL; ++ struct list_head *first; ++ ++ if (list_empty(conf->inactive_list + hash)) ++ goto out; ++ first = (conf->inactive_list + hash)->next; ++ sh = list_entry(first, struct stripe_head, lru); ++ list_del_init(first); ++ remove_hash(sh); ++ atomic_inc(&conf->active_stripes); ++ BUG_ON(hash != sh->hash_lock_index); ++ if (list_empty(conf->inactive_list + hash)) ++ atomic_inc(&conf->empty_inactive_list_nr); ++out: ++ return sh; ++} ++ ++static void shrink_buffers(struct stripe_head *sh) ++{ ++ struct page *p; ++ int i; ++ int num = sh->raid_conf->pool_size; ++ ++ for (i = 0; i < num ; i++) { ++ WARN_ON(sh->dev[i].page != sh->dev[i].orig_page); ++ p = sh->dev[i].page; ++ if (!p) ++ continue; ++ sh->dev[i].page = NULL; ++ put_page(p); ++ } ++} ++ ++static int grow_buffers(struct stripe_head *sh, gfp_t gfp) ++{ ++ int i; ++ int num = sh->raid_conf->pool_size; ++ ++ for (i = 0; i < num; i++) { ++ struct page *page; ++ ++ if (!(page = alloc_page(gfp))) { ++ return 1; ++ } ++ sh->dev[i].page = page; ++ sh->dev[i].orig_page = page; ++ } ++ return 0; ++} ++ ++static void raid5_build_block(struct stripe_head *sh, int i, int previous); ++static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, ++ struct stripe_head *sh); ++ ++static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ int i, seq; ++ ++ BUG_ON(atomic_read(&sh->count) != 0); ++ BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); ++ BUG_ON(stripe_operations_active(sh)); ++ BUG_ON(sh->batch_head); ++ ++ pr_debug("init_stripe called, stripe %llu\n", ++ (unsigned long long)sector); ++retry: ++ seq = read_seqcount_begin(&conf->gen_lock); ++ sh->generation = conf->generation - previous; ++ sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; ++ sh->sector = sector; ++ stripe_set_idx(sector, conf, previous, sh); ++ sh->state = 0; ++ ++ for (i = sh->disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ ++ if (dev->toread || dev->read || dev->towrite || dev->written || ++ test_bit(R5_LOCKED, &dev->flags)) { ++ printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", ++ (unsigned long long)sh->sector, i, dev->toread, ++ dev->read, dev->towrite, dev->written, ++ test_bit(R5_LOCKED, &dev->flags)); ++ WARN_ON(1); ++ } ++ dev->flags = 0; ++ raid5_build_block(sh, i, previous); ++ } ++ if (read_seqcount_retry(&conf->gen_lock, seq)) ++ goto retry; ++ sh->overwrite_disks = 0; ++ insert_hash(conf, sh); ++ sh->cpu = smp_processor_id(); ++ set_bit(STRIPE_BATCH_READY, &sh->state); ++} ++ ++static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, ++ short generation) ++{ ++ struct stripe_head *sh; ++ ++ pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); ++ hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) ++ if (sh->sector == sector && sh->generation == generation) ++ return sh; ++ pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); ++ return NULL; ++} ++ ++/* ++ * Need to check if array has failed when deciding whether to: ++ * - start an array ++ * - remove non-faulty devices ++ * - add a spare ++ * - allow a reshape ++ * This determination is simple when no reshape is happening. ++ * However if there is a reshape, we need to carefully check ++ * both the before and after sections. ++ * This is because some failed devices may only affect one ++ * of the two sections, and some non-in_sync devices may ++ * be insync in the section most affected by failed devices. ++ */ ++static int calc_degraded(struct r5conf *conf) ++{ ++ int degraded, degraded2; ++ int i; ++ ++ rcu_read_lock(); ++ degraded = 0; ++ for (i = 0; i < conf->previous_raid_disks; i++) { ++ struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); ++ if (rdev && test_bit(Faulty, &rdev->flags)) ++ rdev = rcu_dereference(conf->disks[i].replacement); ++ if (!rdev || test_bit(Faulty, &rdev->flags)) ++ degraded++; ++ else if (test_bit(In_sync, &rdev->flags)) ++ ; ++ else ++ /* not in-sync or faulty. ++ * If the reshape increases the number of devices, ++ * this is being recovered by the reshape, so ++ * this 'previous' section is not in_sync. ++ * If the number of devices is being reduced however, ++ * the device can only be part of the array if ++ * we are reverting a reshape, so this section will ++ * be in-sync. ++ */ ++ if (conf->raid_disks >= conf->previous_raid_disks) ++ degraded++; ++ } ++ rcu_read_unlock(); ++ if (conf->raid_disks == conf->previous_raid_disks) ++ return degraded; ++ rcu_read_lock(); ++ degraded2 = 0; ++ for (i = 0; i < conf->raid_disks; i++) { ++ struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); ++ if (rdev && test_bit(Faulty, &rdev->flags)) ++ rdev = rcu_dereference(conf->disks[i].replacement); ++ if (!rdev || test_bit(Faulty, &rdev->flags)) ++ degraded2++; ++ else if (test_bit(In_sync, &rdev->flags)) ++ ; ++ else ++ /* not in-sync or faulty. ++ * If reshape increases the number of devices, this ++ * section has already been recovered, else it ++ * almost certainly hasn't. ++ */ ++ if (conf->raid_disks <= conf->previous_raid_disks) ++ degraded2++; ++ } ++ rcu_read_unlock(); ++ if (degraded2 > degraded) ++ return degraded2; ++ return degraded; ++} ++ ++static int has_failed(struct r5conf *conf) ++{ ++ int degraded; ++ ++ if (conf->mddev->reshape_position == MaxSector) ++ return conf->mddev->degraded > conf->max_degraded; ++ ++ degraded = calc_degraded(conf); ++ if (degraded > conf->max_degraded) ++ return 1; ++ return 0; ++} ++ ++static struct stripe_head * ++get_active_stripe(struct r5conf *conf, sector_t sector, ++ int previous, int noblock, int noquiesce) ++{ ++ struct stripe_head *sh; ++ int hash = stripe_hash_locks_hash(sector); ++ ++ pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); ++ ++ spin_lock_irq(conf->hash_locks + hash); ++ ++ do { ++ wait_event_lock_irq(conf->wait_for_stripe, ++ conf->quiesce == 0 || noquiesce, ++ *(conf->hash_locks + hash)); ++ sh = __find_stripe(conf, sector, conf->generation - previous); ++ if (!sh) { ++ if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) { ++ sh = get_free_stripe(conf, hash); ++ if (!sh && llist_empty(&conf->released_stripes) && ++ !test_bit(R5_DID_ALLOC, &conf->cache_state)) ++ set_bit(R5_ALLOC_MORE, ++ &conf->cache_state); ++ } ++ if (noblock && sh == NULL) ++ break; ++ if (!sh) { ++ set_bit(R5_INACTIVE_BLOCKED, ++ &conf->cache_state); ++ wait_event_lock_irq( ++ conf->wait_for_stripe, ++ !list_empty(conf->inactive_list + hash) && ++ (atomic_read(&conf->active_stripes) ++ < (conf->max_nr_stripes * 3 / 4) ++ || !test_bit(R5_INACTIVE_BLOCKED, ++ &conf->cache_state)), ++ *(conf->hash_locks + hash)); ++ clear_bit(R5_INACTIVE_BLOCKED, ++ &conf->cache_state); ++ } else { ++ init_stripe(sh, sector, previous); ++ atomic_inc(&sh->count); ++ } ++ } else if (!atomic_inc_not_zero(&sh->count)) { ++ spin_lock(&conf->device_lock); ++ if (!atomic_read(&sh->count)) { ++ if (!test_bit(STRIPE_HANDLE, &sh->state)) ++ atomic_inc(&conf->active_stripes); ++ BUG_ON(list_empty(&sh->lru) && ++ !test_bit(STRIPE_EXPANDING, &sh->state)); ++ list_del_init(&sh->lru); ++ if (sh->group) { ++ sh->group->stripes_cnt--; ++ sh->group = NULL; ++ } ++ } ++ atomic_inc(&sh->count); ++ spin_unlock(&conf->device_lock); ++ } ++ } while (sh == NULL); ++ ++ spin_unlock_irq(conf->hash_locks + hash); ++ return sh; ++} ++ ++static bool is_full_stripe_write(struct stripe_head *sh) ++{ ++ BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded)); ++ return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded); ++} ++ ++static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) ++{ ++ local_irq_disable(); ++ if (sh1 > sh2) { ++ spin_lock(&sh2->stripe_lock); ++ spin_lock_nested(&sh1->stripe_lock, 1); ++ } else { ++ spin_lock(&sh1->stripe_lock); ++ spin_lock_nested(&sh2->stripe_lock, 1); ++ } ++} ++ ++static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) ++{ ++ spin_unlock(&sh1->stripe_lock); ++ spin_unlock(&sh2->stripe_lock); ++ local_irq_enable(); ++} ++ ++/* Only freshly new full stripe normal write stripe can be added to a batch list */ ++static bool stripe_can_batch(struct stripe_head *sh) ++{ ++ return test_bit(STRIPE_BATCH_READY, &sh->state) && ++ !test_bit(STRIPE_BITMAP_PENDING, &sh->state) && ++ is_full_stripe_write(sh); ++} ++ ++/* we only do back search */ ++static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh) ++{ ++ struct stripe_head *head; ++ sector_t head_sector, tmp_sec; ++ int hash; ++ int dd_idx; ++ ++ if (!stripe_can_batch(sh)) ++ return; ++ /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */ ++ tmp_sec = sh->sector; ++ if (!sector_div(tmp_sec, conf->chunk_sectors)) ++ return; ++ head_sector = sh->sector - STRIPE_SECTORS; ++ ++ hash = stripe_hash_locks_hash(head_sector); ++ spin_lock_irq(conf->hash_locks + hash); ++ head = __find_stripe(conf, head_sector, conf->generation); ++ if (head && !atomic_inc_not_zero(&head->count)) { ++ spin_lock(&conf->device_lock); ++ if (!atomic_read(&head->count)) { ++ if (!test_bit(STRIPE_HANDLE, &head->state)) ++ atomic_inc(&conf->active_stripes); ++ BUG_ON(list_empty(&head->lru) && ++ !test_bit(STRIPE_EXPANDING, &head->state)); ++ list_del_init(&head->lru); ++ if (head->group) { ++ head->group->stripes_cnt--; ++ head->group = NULL; ++ } ++ } ++ atomic_inc(&head->count); ++ spin_unlock(&conf->device_lock); ++ } ++ spin_unlock_irq(conf->hash_locks + hash); ++ ++ if (!head) ++ return; ++ if (!stripe_can_batch(head)) ++ goto out; ++ ++ lock_two_stripes(head, sh); ++ /* clear_batch_ready clear the flag */ ++ if (!stripe_can_batch(head) || !stripe_can_batch(sh)) ++ goto unlock_out; ++ ++ if (sh->batch_head) ++ goto unlock_out; ++ ++ dd_idx = 0; ++ while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) ++ dd_idx++; ++ if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw) ++ goto unlock_out; ++ ++ if (head->batch_head) { ++ spin_lock(&head->batch_head->batch_lock); ++ /* This batch list is already running */ ++ if (!stripe_can_batch(head)) { ++ spin_unlock(&head->batch_head->batch_lock); ++ goto unlock_out; ++ } ++ ++ /* ++ * at this point, head's BATCH_READY could be cleared, but we ++ * can still add the stripe to batch list ++ */ ++ list_add(&sh->batch_list, &head->batch_list); ++ spin_unlock(&head->batch_head->batch_lock); ++ ++ sh->batch_head = head->batch_head; ++ } else { ++ head->batch_head = head; ++ sh->batch_head = head->batch_head; ++ spin_lock(&head->batch_lock); ++ list_add_tail(&sh->batch_list, &head->batch_list); ++ spin_unlock(&head->batch_lock); ++ } ++ ++ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ if (atomic_dec_return(&conf->preread_active_stripes) ++ < IO_THRESHOLD) ++ md_wakeup_thread(conf->mddev->thread); ++ ++ if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) { ++ int seq = sh->bm_seq; ++ if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) && ++ sh->batch_head->bm_seq > seq) ++ seq = sh->batch_head->bm_seq; ++ set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state); ++ sh->batch_head->bm_seq = seq; ++ } ++ ++ atomic_inc(&sh->count); ++unlock_out: ++ unlock_two_stripes(head, sh); ++out: ++ release_stripe(head); ++} ++ ++/* Determine if 'data_offset' or 'new_data_offset' should be used ++ * in this stripe_head. ++ */ ++static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) ++{ ++ sector_t progress = conf->reshape_progress; ++ /* Need a memory barrier to make sure we see the value ++ * of conf->generation, or ->data_offset that was set before ++ * reshape_progress was updated. ++ */ ++ smp_rmb(); ++ if (progress == MaxSector) ++ return 0; ++ if (sh->generation == conf->generation - 1) ++ return 0; ++ /* We are in a reshape, and this is a new-generation stripe, ++ * so use new_data_offset. ++ */ ++ return 1; ++} ++ ++static void ++raid5_end_read_request(struct bio *bi, int error); ++static void ++raid5_end_write_request(struct bio *bi, int error); ++ ++static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ int i, disks = sh->disks; ++ struct stripe_head *head_sh = sh; ++ ++ might_sleep(); ++ ++ for (i = disks; i--; ) { ++ int rw; ++ int replace_only = 0; ++ struct bio *bi, *rbi; ++ struct md_rdev *rdev, *rrdev = NULL; ++ ++ sh = head_sh; ++ if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { ++ if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) ++ rw = WRITE_FUA; ++ else ++ rw = WRITE; ++ if (test_bit(R5_Discard, &sh->dev[i].flags)) ++ rw |= REQ_DISCARD; ++ } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) ++ rw = READ; ++ else if (test_and_clear_bit(R5_WantReplace, ++ &sh->dev[i].flags)) { ++ rw = WRITE; ++ replace_only = 1; ++ } else ++ continue; ++ if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) ++ rw |= REQ_SYNC; ++ ++again: ++ bi = &sh->dev[i].req; ++ rbi = &sh->dev[i].rreq; /* For writing to replacement */ ++ ++ rcu_read_lock(); ++ rrdev = rcu_dereference(conf->disks[i].replacement); ++ smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ ++ rdev = rcu_dereference(conf->disks[i].rdev); ++ if (!rdev) { ++ rdev = rrdev; ++ rrdev = NULL; ++ } ++ if (rw & WRITE) { ++ if (replace_only) ++ rdev = NULL; ++ if (rdev == rrdev) ++ /* We raced and saw duplicates */ ++ rrdev = NULL; ++ } else { ++ if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev) ++ rdev = rrdev; ++ rrdev = NULL; ++ } ++ ++ if (rdev && test_bit(Faulty, &rdev->flags)) ++ rdev = NULL; ++ if (rdev) ++ atomic_inc(&rdev->nr_pending); ++ if (rrdev && test_bit(Faulty, &rrdev->flags)) ++ rrdev = NULL; ++ if (rrdev) ++ atomic_inc(&rrdev->nr_pending); ++ rcu_read_unlock(); ++ ++ /* We have already checked bad blocks for reads. Now ++ * need to check for writes. We never accept write errors ++ * on the replacement, so we don't to check rrdev. ++ */ ++ while ((rw & WRITE) && rdev && ++ test_bit(WriteErrorSeen, &rdev->flags)) { ++ sector_t first_bad; ++ int bad_sectors; ++ int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, ++ &first_bad, &bad_sectors); ++ if (!bad) ++ break; ++ ++ if (bad < 0) { ++ set_bit(BlockedBadBlocks, &rdev->flags); ++ if (!conf->mddev->external && ++ conf->mddev->flags) { ++ /* It is very unlikely, but we might ++ * still need to write out the ++ * bad block log - better give it ++ * a chance*/ ++ md_check_recovery(conf->mddev); ++ } ++ /* ++ * Because md_wait_for_blocked_rdev ++ * will dec nr_pending, we must ++ * increment it first. ++ */ ++ atomic_inc(&rdev->nr_pending); ++ md_wait_for_blocked_rdev(rdev, conf->mddev); ++ } else { ++ /* Acknowledged bad block - skip the write */ ++ rdev_dec_pending(rdev, conf->mddev); ++ rdev = NULL; ++ } ++ } ++ ++ if (rdev) { ++ if (s->syncing || s->expanding || s->expanded ++ || s->replacing) ++ md_sync_acct(rdev->bdev, STRIPE_SECTORS); ++ ++ set_bit(STRIPE_IO_STARTED, &sh->state); ++ ++ bio_reset(bi); ++ bi->bi_bdev = rdev->bdev; ++ bi->bi_rw = rw; ++ bi->bi_end_io = (rw & WRITE) ++ ? raid5_end_write_request ++ : raid5_end_read_request; ++ bi->bi_private = sh; ++ ++ pr_debug("%s: for %llu schedule op %ld on disc %d\n", ++ __func__, (unsigned long long)sh->sector, ++ bi->bi_rw, i); ++ atomic_inc(&sh->count); ++ if (sh != head_sh) ++ atomic_inc(&head_sh->count); ++ if (use_new_offset(conf, sh)) ++ bi->bi_iter.bi_sector = (sh->sector ++ + rdev->new_data_offset); ++ else ++ bi->bi_iter.bi_sector = (sh->sector ++ + rdev->data_offset); ++ if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags)) ++ bi->bi_rw |= REQ_NOMERGE; ++ ++ if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) ++ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); ++ sh->dev[i].vec.bv_page = sh->dev[i].page; ++ bi->bi_vcnt = 1; ++ bi->bi_io_vec[0].bv_len = STRIPE_SIZE; ++ bi->bi_io_vec[0].bv_offset = 0; ++ bi->bi_iter.bi_size = STRIPE_SIZE; ++ /* ++ * If this is discard request, set bi_vcnt 0. We don't ++ * want to confuse SCSI because SCSI will replace payload ++ */ ++ if (rw & REQ_DISCARD) ++ bi->bi_vcnt = 0; ++ if (rrdev) ++ set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); ++ ++ if (conf->mddev->gendisk) ++ trace_block_bio_remap(bdev_get_queue(bi->bi_bdev), ++ bi, disk_devt(conf->mddev->gendisk), ++ sh->dev[i].sector); ++ generic_make_request(bi); ++ } ++ if (rrdev) { ++ if (s->syncing || s->expanding || s->expanded ++ || s->replacing) ++ md_sync_acct(rrdev->bdev, STRIPE_SECTORS); ++ ++ set_bit(STRIPE_IO_STARTED, &sh->state); ++ ++ bio_reset(rbi); ++ rbi->bi_bdev = rrdev->bdev; ++ rbi->bi_rw = rw; ++ BUG_ON(!(rw & WRITE)); ++ rbi->bi_end_io = raid5_end_write_request; ++ rbi->bi_private = sh; ++ ++ pr_debug("%s: for %llu schedule op %ld on " ++ "replacement disc %d\n", ++ __func__, (unsigned long long)sh->sector, ++ rbi->bi_rw, i); ++ atomic_inc(&sh->count); ++ if (sh != head_sh) ++ atomic_inc(&head_sh->count); ++ if (use_new_offset(conf, sh)) ++ rbi->bi_iter.bi_sector = (sh->sector ++ + rrdev->new_data_offset); ++ else ++ rbi->bi_iter.bi_sector = (sh->sector ++ + rrdev->data_offset); ++ if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) ++ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); ++ sh->dev[i].rvec.bv_page = sh->dev[i].page; ++ rbi->bi_vcnt = 1; ++ rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; ++ rbi->bi_io_vec[0].bv_offset = 0; ++ rbi->bi_iter.bi_size = STRIPE_SIZE; ++ /* ++ * If this is discard request, set bi_vcnt 0. We don't ++ * want to confuse SCSI because SCSI will replace payload ++ */ ++ if (rw & REQ_DISCARD) ++ rbi->bi_vcnt = 0; ++ if (conf->mddev->gendisk) ++ trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), ++ rbi, disk_devt(conf->mddev->gendisk), ++ sh->dev[i].sector); ++ generic_make_request(rbi); ++ } ++ if (!rdev && !rrdev) { ++ if (rw & WRITE) ++ set_bit(STRIPE_DEGRADED, &sh->state); ++ pr_debug("skip op %ld on disc %d for sector %llu\n", ++ bi->bi_rw, i, (unsigned long long)sh->sector); ++ clear_bit(R5_LOCKED, &sh->dev[i].flags); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ } ++ ++ if (!head_sh->batch_head) ++ continue; ++ sh = list_first_entry(&sh->batch_list, struct stripe_head, ++ batch_list); ++ if (sh != head_sh) ++ goto again; ++ } ++} ++ ++static struct dma_async_tx_descriptor * ++async_copy_data(int frombio, struct bio *bio, struct page **page, ++ sector_t sector, struct dma_async_tx_descriptor *tx, ++ struct stripe_head *sh) ++{ ++ struct bio_vec bvl; ++ struct bvec_iter iter; ++ struct page *bio_page; ++ int page_offset; ++ struct async_submit_ctl submit; ++ enum async_tx_flags flags = 0; ++ ++ if (bio->bi_iter.bi_sector >= sector) ++ page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512; ++ else ++ page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512; ++ ++ if (frombio) ++ flags |= ASYNC_TX_FENCE; ++ init_async_submit(&submit, flags, tx, NULL, NULL, NULL); ++ ++ bio_for_each_segment(bvl, bio, iter) { ++ int len = bvl.bv_len; ++ int clen; ++ int b_offset = 0; ++ ++ if (page_offset < 0) { ++ b_offset = -page_offset; ++ page_offset += b_offset; ++ len -= b_offset; ++ } ++ ++ if (len > 0 && page_offset + len > STRIPE_SIZE) ++ clen = STRIPE_SIZE - page_offset; ++ else ++ clen = len; ++ ++ if (clen > 0) { ++ b_offset += bvl.bv_offset; ++ bio_page = bvl.bv_page; ++ if (frombio) { ++ if (sh->raid_conf->skip_copy && ++ b_offset == 0 && page_offset == 0 && ++ clen == STRIPE_SIZE) ++ *page = bio_page; ++ else ++ tx = async_memcpy(*page, bio_page, page_offset, ++ b_offset, clen, &submit); ++ } else ++ tx = async_memcpy(bio_page, *page, b_offset, ++ page_offset, clen, &submit); ++ } ++ /* chain the operations */ ++ submit.depend_tx = tx; ++ ++ if (clen < len) /* hit end of page */ ++ break; ++ page_offset += len; ++ } ++ ++ return tx; ++} ++ ++static void ops_complete_biofill(void *stripe_head_ref) ++{ ++ struct stripe_head *sh = stripe_head_ref; ++ struct bio *return_bi = NULL; ++ int i; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ /* clear completed biofills */ ++ for (i = sh->disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ ++ /* acknowledge completion of a biofill operation */ ++ /* and check if we need to reply to a read request, ++ * new R5_Wantfill requests are held off until ++ * !STRIPE_BIOFILL_RUN ++ */ ++ if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { ++ struct bio *rbi, *rbi2; ++ ++ BUG_ON(!dev->read); ++ rbi = dev->read; ++ dev->read = NULL; ++ while (rbi && rbi->bi_iter.bi_sector < ++ dev->sector + STRIPE_SECTORS) { ++ rbi2 = r5_next_bio(rbi, dev->sector); ++ if (!raid5_dec_bi_active_stripes(rbi)) { ++ rbi->bi_next = return_bi; ++ return_bi = rbi; ++ } ++ rbi = rbi2; ++ } ++ } ++ } ++ clear_bit(STRIPE_BIOFILL_RUN, &sh->state); ++ ++ return_io(return_bi); ++ ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++} ++ ++static void ops_run_biofill(struct stripe_head *sh) ++{ ++ struct dma_async_tx_descriptor *tx = NULL; ++ struct async_submit_ctl submit; ++ int i; ++ ++ BUG_ON(sh->batch_head); ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ for (i = sh->disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (test_bit(R5_Wantfill, &dev->flags)) { ++ struct bio *rbi; ++ spin_lock_irq(&sh->stripe_lock); ++ dev->read = rbi = dev->toread; ++ dev->toread = NULL; ++ spin_unlock_irq(&sh->stripe_lock); ++ while (rbi && rbi->bi_iter.bi_sector < ++ dev->sector + STRIPE_SECTORS) { ++ tx = async_copy_data(0, rbi, &dev->page, ++ dev->sector, tx, sh); ++ rbi = r5_next_bio(rbi, dev->sector); ++ } ++ } ++ } ++ ++ atomic_inc(&sh->count); ++ init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); ++ async_trigger_callback(&submit); ++} ++ ++static void mark_target_uptodate(struct stripe_head *sh, int target) ++{ ++ struct r5dev *tgt; ++ ++ if (target < 0) ++ return; ++ ++ tgt = &sh->dev[target]; ++ set_bit(R5_UPTODATE, &tgt->flags); ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); ++ clear_bit(R5_Wantcompute, &tgt->flags); ++} ++ ++static void ops_complete_compute(void *stripe_head_ref) ++{ ++ struct stripe_head *sh = stripe_head_ref; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ /* mark the computed target(s) as uptodate */ ++ mark_target_uptodate(sh, sh->ops.target); ++ mark_target_uptodate(sh, sh->ops.target2); ++ ++ clear_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ if (sh->check_state == check_state_compute_run) ++ sh->check_state = check_state_compute_result; ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++} ++ ++/* return a pointer to the address conversion region of the scribble buffer */ ++static addr_conv_t *to_addr_conv(struct stripe_head *sh, ++ struct raid5_percpu *percpu, int i) ++{ ++ void *addr; ++ ++ addr = flex_array_get(percpu->scribble, i); ++ return addr + sizeof(struct page *) * (sh->disks + 2); ++} ++ ++/* return a pointer to the address conversion region of the scribble buffer */ ++static struct page **to_addr_page(struct raid5_percpu *percpu, int i) ++{ ++ void *addr; ++ ++ addr = flex_array_get(percpu->scribble, i); ++ return addr; ++} ++ ++static struct dma_async_tx_descriptor * ++ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) ++{ ++ int disks = sh->disks; ++ struct page **xor_srcs = to_addr_page(percpu, 0); ++ int target = sh->ops.target; ++ struct r5dev *tgt = &sh->dev[target]; ++ struct page *xor_dest = tgt->page; ++ int count = 0; ++ struct dma_async_tx_descriptor *tx; ++ struct async_submit_ctl submit; ++ int i; ++ ++ BUG_ON(sh->batch_head); ++ ++ pr_debug("%s: stripe %llu block: %d\n", ++ __func__, (unsigned long long)sh->sector, target); ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); ++ ++ for (i = disks; i--; ) ++ if (i != target) ++ xor_srcs[count++] = sh->dev[i].page; ++ ++ atomic_inc(&sh->count); ++ ++ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, ++ ops_complete_compute, sh, to_addr_conv(sh, percpu, 0)); ++ if (unlikely(count == 1)) ++ tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); ++ else ++ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); ++ ++ return tx; ++} ++ ++/* set_syndrome_sources - populate source buffers for gen_syndrome ++ * @srcs - (struct page *) array of size sh->disks ++ * @sh - stripe_head to parse ++ * ++ * Populates srcs in proper layout order for the stripe and returns the ++ * 'count' of sources to be used in a call to async_gen_syndrome. The P ++ * destination buffer is recorded in srcs[count] and the Q destination ++ * is recorded in srcs[count+1]]. ++ */ ++static int set_syndrome_sources(struct page **srcs, ++ struct stripe_head *sh, ++ int srctype) ++{ ++ int disks = sh->disks; ++ int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); ++ int d0_idx = raid6_d0(sh); ++ int count; ++ int i; ++ ++ for (i = 0; i < disks; i++) ++ srcs[i] = NULL; ++ ++ count = 0; ++ i = d0_idx; ++ do { ++ int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); ++ struct r5dev *dev = &sh->dev[i]; ++ ++ if (i == sh->qd_idx || i == sh->pd_idx || ++ (srctype == SYNDROME_SRC_ALL) || ++ (srctype == SYNDROME_SRC_WANT_DRAIN && ++ test_bit(R5_Wantdrain, &dev->flags)) || ++ (srctype == SYNDROME_SRC_WRITTEN && ++ dev->written)) ++ srcs[slot] = sh->dev[i].page; ++ i = raid6_next_disk(i, disks); ++ } while (i != d0_idx); ++ ++ return syndrome_disks; ++} ++ ++static struct dma_async_tx_descriptor * ++ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) ++{ ++ int disks = sh->disks; ++ struct page **blocks = to_addr_page(percpu, 0); ++ int target; ++ int qd_idx = sh->qd_idx; ++ struct dma_async_tx_descriptor *tx; ++ struct async_submit_ctl submit; ++ struct r5dev *tgt; ++ struct page *dest; ++ int i; ++ int count; ++ ++ BUG_ON(sh->batch_head); ++ if (sh->ops.target < 0) ++ target = sh->ops.target2; ++ else if (sh->ops.target2 < 0) ++ target = sh->ops.target; ++ else ++ /* we should only have one valid target */ ++ BUG(); ++ BUG_ON(target < 0); ++ pr_debug("%s: stripe %llu block: %d\n", ++ __func__, (unsigned long long)sh->sector, target); ++ ++ tgt = &sh->dev[target]; ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); ++ dest = tgt->page; ++ ++ atomic_inc(&sh->count); ++ ++ if (target == qd_idx) { ++ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); ++ blocks[count] = NULL; /* regenerating p is not necessary */ ++ BUG_ON(blocks[count+1] != dest); /* q should already be set */ ++ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ++ ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); ++ } else { ++ /* Compute any data- or p-drive using XOR */ ++ count = 0; ++ for (i = disks; i-- ; ) { ++ if (i == target || i == qd_idx) ++ continue; ++ blocks[count++] = sh->dev[i].page; ++ } ++ ++ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, ++ NULL, ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); ++ } ++ ++ return tx; ++} ++ ++static struct dma_async_tx_descriptor * ++ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) ++{ ++ int i, count, disks = sh->disks; ++ int syndrome_disks = sh->ddf_layout ? disks : disks-2; ++ int d0_idx = raid6_d0(sh); ++ int faila = -1, failb = -1; ++ int target = sh->ops.target; ++ int target2 = sh->ops.target2; ++ struct r5dev *tgt = &sh->dev[target]; ++ struct r5dev *tgt2 = &sh->dev[target2]; ++ struct dma_async_tx_descriptor *tx; ++ struct page **blocks = to_addr_page(percpu, 0); ++ struct async_submit_ctl submit; ++ ++ BUG_ON(sh->batch_head); ++ pr_debug("%s: stripe %llu block1: %d block2: %d\n", ++ __func__, (unsigned long long)sh->sector, target, target2); ++ BUG_ON(target < 0 || target2 < 0); ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); ++ ++ /* we need to open-code set_syndrome_sources to handle the ++ * slot number conversion for 'faila' and 'failb' ++ */ ++ for (i = 0; i < disks ; i++) ++ blocks[i] = NULL; ++ count = 0; ++ i = d0_idx; ++ do { ++ int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); ++ ++ blocks[slot] = sh->dev[i].page; ++ ++ if (i == target) ++ faila = slot; ++ if (i == target2) ++ failb = slot; ++ i = raid6_next_disk(i, disks); ++ } while (i != d0_idx); ++ ++ BUG_ON(faila == failb); ++ if (failb < faila) ++ swap(faila, failb); ++ pr_debug("%s: stripe: %llu faila: %d failb: %d\n", ++ __func__, (unsigned long long)sh->sector, faila, failb); ++ ++ atomic_inc(&sh->count); ++ ++ if (failb == syndrome_disks+1) { ++ /* Q disk is one of the missing disks */ ++ if (faila == syndrome_disks) { ++ /* Missing P+Q, just recompute */ ++ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ++ ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ return async_gen_syndrome(blocks, 0, syndrome_disks+2, ++ STRIPE_SIZE, &submit); ++ } else { ++ struct page *dest; ++ int data_target; ++ int qd_idx = sh->qd_idx; ++ ++ /* Missing D+Q: recompute D from P, then recompute Q */ ++ if (target == qd_idx) ++ data_target = target2; ++ else ++ data_target = target; ++ ++ count = 0; ++ for (i = disks; i-- ; ) { ++ if (i == data_target || i == qd_idx) ++ continue; ++ blocks[count++] = sh->dev[i].page; ++ } ++ dest = sh->dev[data_target].page; ++ init_async_submit(&submit, ++ ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, ++ NULL, NULL, NULL, ++ to_addr_conv(sh, percpu, 0)); ++ tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, ++ &submit); ++ ++ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); ++ init_async_submit(&submit, ASYNC_TX_FENCE, tx, ++ ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ return async_gen_syndrome(blocks, 0, count+2, ++ STRIPE_SIZE, &submit); ++ } ++ } else { ++ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ++ ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ if (failb == syndrome_disks) { ++ /* We're missing D+P. */ ++ return async_raid6_datap_recov(syndrome_disks+2, ++ STRIPE_SIZE, faila, ++ blocks, &submit); ++ } else { ++ /* We're missing D+D. */ ++ return async_raid6_2data_recov(syndrome_disks+2, ++ STRIPE_SIZE, faila, failb, ++ blocks, &submit); ++ } ++ } ++} ++ ++static void ops_complete_prexor(void *stripe_head_ref) ++{ ++ struct stripe_head *sh = stripe_head_ref; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++} ++ ++static struct dma_async_tx_descriptor * ++ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu, ++ struct dma_async_tx_descriptor *tx) ++{ ++ int disks = sh->disks; ++ struct page **xor_srcs = to_addr_page(percpu, 0); ++ int count = 0, pd_idx = sh->pd_idx, i; ++ struct async_submit_ctl submit; ++ ++ /* existing parity data subtracted */ ++ struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; ++ ++ BUG_ON(sh->batch_head); ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ /* Only process blocks that are known to be uptodate */ ++ if (test_bit(R5_Wantdrain, &dev->flags)) ++ xor_srcs[count++] = dev->page; ++ } ++ ++ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, ++ ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); ++ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); ++ ++ return tx; ++} ++ ++static struct dma_async_tx_descriptor * ++ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu, ++ struct dma_async_tx_descriptor *tx) ++{ ++ struct page **blocks = to_addr_page(percpu, 0); ++ int count; ++ struct async_submit_ctl submit; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN); ++ ++ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx, ++ ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); ++ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); ++ ++ return tx; ++} ++ ++static struct dma_async_tx_descriptor * ++ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) ++{ ++ int disks = sh->disks; ++ int i; ++ struct stripe_head *head_sh = sh; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ for (i = disks; i--; ) { ++ struct r5dev *dev; ++ struct bio *chosen; ++ ++ sh = head_sh; ++ if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) { ++ struct bio *wbi; ++ ++again: ++ dev = &sh->dev[i]; ++ spin_lock_irq(&sh->stripe_lock); ++ chosen = dev->towrite; ++ dev->towrite = NULL; ++ sh->overwrite_disks = 0; ++ BUG_ON(dev->written); ++ wbi = dev->written = chosen; ++ spin_unlock_irq(&sh->stripe_lock); ++ WARN_ON(dev->page != dev->orig_page); ++ ++ while (wbi && wbi->bi_iter.bi_sector < ++ dev->sector + STRIPE_SECTORS) { ++ if (wbi->bi_rw & REQ_FUA) ++ set_bit(R5_WantFUA, &dev->flags); ++ if (wbi->bi_rw & REQ_SYNC) ++ set_bit(R5_SyncIO, &dev->flags); ++ if (wbi->bi_rw & REQ_DISCARD) ++ set_bit(R5_Discard, &dev->flags); ++ else { ++ tx = async_copy_data(1, wbi, &dev->page, ++ dev->sector, tx, sh); ++ if (dev->page != dev->orig_page) { ++ set_bit(R5_SkipCopy, &dev->flags); ++ clear_bit(R5_UPTODATE, &dev->flags); ++ clear_bit(R5_OVERWRITE, &dev->flags); ++ } ++ } ++ wbi = r5_next_bio(wbi, dev->sector); ++ } ++ ++ if (head_sh->batch_head) { ++ sh = list_first_entry(&sh->batch_list, ++ struct stripe_head, ++ batch_list); ++ if (sh == head_sh) ++ continue; ++ goto again; ++ } ++ } ++ } ++ ++ return tx; ++} ++ ++static void ops_complete_reconstruct(void *stripe_head_ref) ++{ ++ struct stripe_head *sh = stripe_head_ref; ++ int disks = sh->disks; ++ int pd_idx = sh->pd_idx; ++ int qd_idx = sh->qd_idx; ++ int i; ++ bool fua = false, sync = false, discard = false; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ for (i = disks; i--; ) { ++ fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); ++ sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); ++ discard |= test_bit(R5_Discard, &sh->dev[i].flags); ++ } ++ ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ ++ if (dev->written || i == pd_idx || i == qd_idx) { ++ if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) ++ set_bit(R5_UPTODATE, &dev->flags); ++ if (fua) ++ set_bit(R5_WantFUA, &dev->flags); ++ if (sync) ++ set_bit(R5_SyncIO, &dev->flags); ++ } ++ } ++ ++ if (sh->reconstruct_state == reconstruct_state_drain_run) ++ sh->reconstruct_state = reconstruct_state_drain_result; ++ else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) ++ sh->reconstruct_state = reconstruct_state_prexor_drain_result; ++ else { ++ BUG_ON(sh->reconstruct_state != reconstruct_state_run); ++ sh->reconstruct_state = reconstruct_state_result; ++ } ++ ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++} ++ ++static void ++ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, ++ struct dma_async_tx_descriptor *tx) ++{ ++ int disks = sh->disks; ++ struct page **xor_srcs; ++ struct async_submit_ctl submit; ++ int count, pd_idx = sh->pd_idx, i; ++ struct page *xor_dest; ++ int prexor = 0; ++ unsigned long flags; ++ int j = 0; ++ struct stripe_head *head_sh = sh; ++ int last_stripe; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ for (i = 0; i < sh->disks; i++) { ++ if (pd_idx == i) ++ continue; ++ if (!test_bit(R5_Discard, &sh->dev[i].flags)) ++ break; ++ } ++ if (i >= sh->disks) { ++ atomic_inc(&sh->count); ++ set_bit(R5_Discard, &sh->dev[pd_idx].flags); ++ ops_complete_reconstruct(sh); ++ return; ++ } ++again: ++ count = 0; ++ xor_srcs = to_addr_page(percpu, j); ++ /* check if prexor is active which means only process blocks ++ * that are part of a read-modify-write (written) ++ */ ++ if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) { ++ prexor = 1; ++ xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (head_sh->dev[i].written) ++ xor_srcs[count++] = dev->page; ++ } ++ } else { ++ xor_dest = sh->dev[pd_idx].page; ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (i != pd_idx) ++ xor_srcs[count++] = dev->page; ++ } ++ } ++ ++ /* 1/ if we prexor'd then the dest is reused as a source ++ * 2/ if we did not prexor then we are redoing the parity ++ * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST ++ * for the synchronous xor case ++ */ ++ last_stripe = !head_sh->batch_head || ++ list_first_entry(&sh->batch_list, ++ struct stripe_head, batch_list) == head_sh; ++ if (last_stripe) { ++ flags = ASYNC_TX_ACK | ++ (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); ++ ++ atomic_inc(&head_sh->count); ++ init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh, ++ to_addr_conv(sh, percpu, j)); ++ } else { ++ flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST; ++ init_async_submit(&submit, flags, tx, NULL, NULL, ++ to_addr_conv(sh, percpu, j)); ++ } ++ ++ if (unlikely(count == 1)) ++ tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); ++ else ++ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); ++ if (!last_stripe) { ++ j++; ++ sh = list_first_entry(&sh->batch_list, struct stripe_head, ++ batch_list); ++ goto again; ++ } ++} ++ ++static void ++ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, ++ struct dma_async_tx_descriptor *tx) ++{ ++ struct async_submit_ctl submit; ++ struct page **blocks; ++ int count, i, j = 0; ++ struct stripe_head *head_sh = sh; ++ int last_stripe; ++ int synflags; ++ unsigned long txflags; ++ ++ pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); ++ ++ for (i = 0; i < sh->disks; i++) { ++ if (sh->pd_idx == i || sh->qd_idx == i) ++ continue; ++ if (!test_bit(R5_Discard, &sh->dev[i].flags)) ++ break; ++ } ++ if (i >= sh->disks) { ++ atomic_inc(&sh->count); ++ set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); ++ set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); ++ ops_complete_reconstruct(sh); ++ return; ++ } ++ ++again: ++ blocks = to_addr_page(percpu, j); ++ ++ if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { ++ synflags = SYNDROME_SRC_WRITTEN; ++ txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST; ++ } else { ++ synflags = SYNDROME_SRC_ALL; ++ txflags = ASYNC_TX_ACK; ++ } ++ ++ count = set_syndrome_sources(blocks, sh, synflags); ++ last_stripe = !head_sh->batch_head || ++ list_first_entry(&sh->batch_list, ++ struct stripe_head, batch_list) == head_sh; ++ ++ if (last_stripe) { ++ atomic_inc(&head_sh->count); ++ init_async_submit(&submit, txflags, tx, ops_complete_reconstruct, ++ head_sh, to_addr_conv(sh, percpu, j)); ++ } else ++ init_async_submit(&submit, 0, tx, NULL, NULL, ++ to_addr_conv(sh, percpu, j)); ++ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); ++ if (!last_stripe) { ++ j++; ++ sh = list_first_entry(&sh->batch_list, struct stripe_head, ++ batch_list); ++ goto again; ++ } ++} ++ ++static void ops_complete_check(void *stripe_head_ref) ++{ ++ struct stripe_head *sh = stripe_head_ref; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ sh->check_state = check_state_check_result; ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++} ++ ++static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) ++{ ++ int disks = sh->disks; ++ int pd_idx = sh->pd_idx; ++ int qd_idx = sh->qd_idx; ++ struct page *xor_dest; ++ struct page **xor_srcs = to_addr_page(percpu, 0); ++ struct dma_async_tx_descriptor *tx; ++ struct async_submit_ctl submit; ++ int count; ++ int i; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ BUG_ON(sh->batch_head); ++ count = 0; ++ xor_dest = sh->dev[pd_idx].page; ++ xor_srcs[count++] = xor_dest; ++ for (i = disks; i--; ) { ++ if (i == pd_idx || i == qd_idx) ++ continue; ++ xor_srcs[count++] = sh->dev[i].page; ++ } ++ ++ init_async_submit(&submit, 0, NULL, NULL, NULL, ++ to_addr_conv(sh, percpu, 0)); ++ tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, ++ &sh->ops.zero_sum_result, &submit); ++ ++ atomic_inc(&sh->count); ++ init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); ++ tx = async_trigger_callback(&submit); ++} ++ ++static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) ++{ ++ struct page **srcs = to_addr_page(percpu, 0); ++ struct async_submit_ctl submit; ++ int count; ++ ++ pr_debug("%s: stripe %llu checkp: %d\n", __func__, ++ (unsigned long long)sh->sector, checkp); ++ ++ BUG_ON(sh->batch_head); ++ count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL); ++ if (!checkp) ++ srcs[count] = NULL; ++ ++ atomic_inc(&sh->count); ++ init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, ++ sh, to_addr_conv(sh, percpu, 0)); ++ async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, ++ &sh->ops.zero_sum_result, percpu->spare_page, &submit); ++} ++ ++static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) ++{ ++ int overlap_clear = 0, i, disks = sh->disks; ++ struct dma_async_tx_descriptor *tx = NULL; ++ struct r5conf *conf = sh->raid_conf; ++ int level = conf->level; ++ struct raid5_percpu *percpu; ++ unsigned long cpu; ++ ++ cpu = get_cpu(); ++ percpu = per_cpu_ptr(conf->percpu, cpu); ++ if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { ++ ops_run_biofill(sh); ++ overlap_clear++; ++ } ++ ++ if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { ++ if (level < 6) ++ tx = ops_run_compute5(sh, percpu); ++ else { ++ if (sh->ops.target2 < 0 || sh->ops.target < 0) ++ tx = ops_run_compute6_1(sh, percpu); ++ else ++ tx = ops_run_compute6_2(sh, percpu); ++ } ++ /* terminate the chain if reconstruct is not set to be run */ ++ if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) ++ async_tx_ack(tx); ++ } ++ ++ if (test_bit(STRIPE_OP_PREXOR, &ops_request)) { ++ if (level < 6) ++ tx = ops_run_prexor5(sh, percpu, tx); ++ else ++ tx = ops_run_prexor6(sh, percpu, tx); ++ } ++ ++ if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { ++ tx = ops_run_biodrain(sh, tx); ++ overlap_clear++; ++ } ++ ++ if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { ++ if (level < 6) ++ ops_run_reconstruct5(sh, percpu, tx); ++ else ++ ops_run_reconstruct6(sh, percpu, tx); ++ } ++ ++ if (test_bit(STRIPE_OP_CHECK, &ops_request)) { ++ if (sh->check_state == check_state_run) ++ ops_run_check_p(sh, percpu); ++ else if (sh->check_state == check_state_run_q) ++ ops_run_check_pq(sh, percpu, 0); ++ else if (sh->check_state == check_state_run_pq) ++ ops_run_check_pq(sh, percpu, 1); ++ else ++ BUG(); ++ } ++ ++ if (overlap_clear && !sh->batch_head) ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (test_and_clear_bit(R5_Overlap, &dev->flags)) ++ wake_up(&sh->raid_conf->wait_for_overlap); ++ } ++ put_cpu(); ++} ++ ++static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp) ++{ ++ struct stripe_head *sh; ++ ++ sh = kmem_cache_zalloc(sc, gfp); ++ if (sh) { ++ spin_lock_init(&sh->stripe_lock); ++ spin_lock_init(&sh->batch_lock); ++ INIT_LIST_HEAD(&sh->batch_list); ++ INIT_LIST_HEAD(&sh->lru); ++ atomic_set(&sh->count, 1); ++ } ++ return sh; ++} ++static int grow_one_stripe(struct r5conf *conf, gfp_t gfp) ++{ ++ struct stripe_head *sh; ++ ++ sh = alloc_stripe(conf->slab_cache, gfp); ++ if (!sh) ++ return 0; ++ ++ sh->raid_conf = conf; ++ ++ if (grow_buffers(sh, gfp)) { ++ shrink_buffers(sh); ++ kmem_cache_free(conf->slab_cache, sh); ++ return 0; ++ } ++ sh->hash_lock_index = ++ conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; ++ /* we just created an active stripe so... */ ++ atomic_inc(&conf->active_stripes); ++ ++ release_stripe(sh); ++ conf->max_nr_stripes++; ++ return 1; ++} ++ ++static int grow_stripes(struct r5conf *conf, int num) ++{ ++ struct kmem_cache *sc; ++ int devs = max(conf->raid_disks, conf->previous_raid_disks); ++ ++ if (conf->mddev->gendisk) ++ sprintf(conf->cache_name[0], ++ "raid%d-%s", conf->level, mdname(conf->mddev)); ++ else ++ sprintf(conf->cache_name[0], ++ "raid%d-%p", conf->level, conf->mddev); ++ sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); ++ ++ conf->active_name = 0; ++ sc = kmem_cache_create(conf->cache_name[conf->active_name], ++ sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), ++ 0, 0, NULL); ++ if (!sc) ++ return 1; ++ conf->slab_cache = sc; ++ conf->pool_size = devs; ++ while (num--) ++ if (!grow_one_stripe(conf, GFP_KERNEL)) ++ return 1; ++ ++ return 0; ++} ++ ++/** ++ * scribble_len - return the required size of the scribble region ++ * @num - total number of disks in the array ++ * ++ * The size must be enough to contain: ++ * 1/ a struct page pointer for each device in the array +2 ++ * 2/ room to convert each entry in (1) to its corresponding dma ++ * (dma_map_page()) or page (page_address()) address. ++ * ++ * Note: the +2 is for the destination buffers of the ddf/raid6 case where we ++ * calculate over all devices (not just the data blocks), using zeros in place ++ * of the P and Q blocks. ++ */ ++static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags) ++{ ++ struct flex_array *ret; ++ size_t len; ++ ++ len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); ++ ret = flex_array_alloc(len, cnt, flags); ++ if (!ret) ++ return NULL; ++ /* always prealloc all elements, so no locking is required */ ++ if (flex_array_prealloc(ret, 0, cnt, flags)) { ++ flex_array_free(ret); ++ return NULL; ++ } ++ return ret; ++} ++ ++static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) ++{ ++ unsigned long cpu; ++ int err = 0; ++ ++ mddev_suspend(conf->mddev); ++ get_online_cpus(); ++ for_each_present_cpu(cpu) { ++ struct raid5_percpu *percpu; ++ struct flex_array *scribble; ++ ++ percpu = per_cpu_ptr(conf->percpu, cpu); ++ scribble = scribble_alloc(new_disks, ++ new_sectors / STRIPE_SECTORS, ++ GFP_NOIO); ++ ++ if (scribble) { ++ flex_array_free(percpu->scribble); ++ percpu->scribble = scribble; ++ } else { ++ err = -ENOMEM; ++ break; ++ } ++ } ++ put_online_cpus(); ++ mddev_resume(conf->mddev); ++ return err; ++} ++ ++static int resize_stripes(struct r5conf *conf, int newsize) ++{ ++ /* Make all the stripes able to hold 'newsize' devices. ++ * New slots in each stripe get 'page' set to a new page. ++ * ++ * This happens in stages: ++ * 1/ create a new kmem_cache and allocate the required number of ++ * stripe_heads. ++ * 2/ gather all the old stripe_heads and transfer the pages across ++ * to the new stripe_heads. This will have the side effect of ++ * freezing the array as once all stripe_heads have been collected, ++ * no IO will be possible. Old stripe heads are freed once their ++ * pages have been transferred over, and the old kmem_cache is ++ * freed when all stripes are done. ++ * 3/ reallocate conf->disks to be suitable bigger. If this fails, ++ * we simple return a failre status - no need to clean anything up. ++ * 4/ allocate new pages for the new slots in the new stripe_heads. ++ * If this fails, we don't bother trying the shrink the ++ * stripe_heads down again, we just leave them as they are. ++ * As each stripe_head is processed the new one is released into ++ * active service. ++ * ++ * Once step2 is started, we cannot afford to wait for a write, ++ * so we use GFP_NOIO allocations. ++ */ ++ struct stripe_head *osh, *nsh; ++ LIST_HEAD(newstripes); ++ struct disk_info *ndisks; ++ int err; ++ struct kmem_cache *sc; ++ int i; ++ int hash, cnt; ++ ++ if (newsize <= conf->pool_size) ++ return 0; /* never bother to shrink */ ++ ++ err = md_allow_write(conf->mddev); ++ if (err) ++ return err; ++ ++ /* Step 1 */ ++ sc = kmem_cache_create(conf->cache_name[1-conf->active_name], ++ sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), ++ 0, 0, NULL); ++ if (!sc) ++ return -ENOMEM; ++ ++ /* Need to ensure auto-resizing doesn't interfere */ ++ mutex_lock(&conf->cache_size_mutex); ++ ++ for (i = conf->max_nr_stripes; i; i--) { ++ nsh = alloc_stripe(sc, GFP_KERNEL); ++ if (!nsh) ++ break; ++ ++ nsh->raid_conf = conf; ++ list_add(&nsh->lru, &newstripes); ++ } ++ if (i) { ++ /* didn't get enough, give up */ ++ while (!list_empty(&newstripes)) { ++ nsh = list_entry(newstripes.next, struct stripe_head, lru); ++ list_del(&nsh->lru); ++ kmem_cache_free(sc, nsh); ++ } ++ kmem_cache_destroy(sc); ++ mutex_unlock(&conf->cache_size_mutex); ++ return -ENOMEM; ++ } ++ /* Step 2 - Must use GFP_NOIO now. ++ * OK, we have enough stripes, start collecting inactive ++ * stripes and copying them over ++ */ ++ hash = 0; ++ cnt = 0; ++ list_for_each_entry(nsh, &newstripes, lru) { ++ lock_device_hash_lock(conf, hash); ++ wait_event_cmd(conf->wait_for_stripe, ++ !list_empty(conf->inactive_list + hash), ++ unlock_device_hash_lock(conf, hash), ++ lock_device_hash_lock(conf, hash)); ++ osh = get_free_stripe(conf, hash); ++ unlock_device_hash_lock(conf, hash); ++ ++ for(i=0; i<conf->pool_size; i++) { ++ nsh->dev[i].page = osh->dev[i].page; ++ nsh->dev[i].orig_page = osh->dev[i].page; ++ } ++ nsh->hash_lock_index = hash; ++ kmem_cache_free(conf->slab_cache, osh); ++ cnt++; ++ if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS + ++ !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) { ++ hash++; ++ cnt = 0; ++ } ++ } ++ kmem_cache_destroy(conf->slab_cache); ++ ++ /* Step 3. ++ * At this point, we are holding all the stripes so the array ++ * is completely stalled, so now is a good time to resize ++ * conf->disks and the scribble region ++ */ ++ ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); ++ if (ndisks) { ++ for (i=0; i<conf->raid_disks; i++) ++ ndisks[i] = conf->disks[i]; ++ kfree(conf->disks); ++ conf->disks = ndisks; ++ } else ++ err = -ENOMEM; ++ ++ mutex_unlock(&conf->cache_size_mutex); ++ /* Step 4, return new stripes to service */ ++ while(!list_empty(&newstripes)) { ++ nsh = list_entry(newstripes.next, struct stripe_head, lru); ++ list_del_init(&nsh->lru); ++ ++ for (i=conf->raid_disks; i < newsize; i++) ++ if (nsh->dev[i].page == NULL) { ++ struct page *p = alloc_page(GFP_NOIO); ++ nsh->dev[i].page = p; ++ nsh->dev[i].orig_page = p; ++ if (!p) ++ err = -ENOMEM; ++ } ++ release_stripe(nsh); ++ } ++ /* critical section pass, GFP_NOIO no longer needed */ ++ ++ conf->slab_cache = sc; ++ conf->active_name = 1-conf->active_name; ++ if (!err) ++ conf->pool_size = newsize; ++ return err; ++} ++ ++static int drop_one_stripe(struct r5conf *conf) ++{ ++ struct stripe_head *sh; ++ int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK; ++ ++ spin_lock_irq(conf->hash_locks + hash); ++ sh = get_free_stripe(conf, hash); ++ spin_unlock_irq(conf->hash_locks + hash); ++ if (!sh) ++ return 0; ++ BUG_ON(atomic_read(&sh->count)); ++ shrink_buffers(sh); ++ kmem_cache_free(conf->slab_cache, sh); ++ atomic_dec(&conf->active_stripes); ++ conf->max_nr_stripes--; ++ return 1; ++} ++ ++static void shrink_stripes(struct r5conf *conf) ++{ ++ while (conf->max_nr_stripes && ++ drop_one_stripe(conf)) ++ ; ++ ++ if (conf->slab_cache) ++ kmem_cache_destroy(conf->slab_cache); ++ conf->slab_cache = NULL; ++} ++ ++static void raid5_end_read_request(struct bio * bi, int error) ++{ ++ struct stripe_head *sh = bi->bi_private; ++ struct r5conf *conf = sh->raid_conf; ++ int disks = sh->disks, i; ++ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); ++ char b[BDEVNAME_SIZE]; ++ struct md_rdev *rdev = NULL; ++ sector_t s; ++ ++ for (i=0 ; i<disks; i++) ++ if (bi == &sh->dev[i].req) ++ break; ++ ++ pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", ++ (unsigned long long)sh->sector, i, atomic_read(&sh->count), ++ uptodate); ++ if (i == disks) { ++ BUG(); ++ return; ++ } ++ if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) ++ /* If replacement finished while this request was outstanding, ++ * 'replacement' might be NULL already. ++ * In that case it moved down to 'rdev'. ++ * rdev is not removed until all requests are finished. ++ */ ++ rdev = conf->disks[i].replacement; ++ if (!rdev) ++ rdev = conf->disks[i].rdev; ++ ++ if (use_new_offset(conf, sh)) ++ s = sh->sector + rdev->new_data_offset; ++ else ++ s = sh->sector + rdev->data_offset; ++ if (uptodate) { ++ set_bit(R5_UPTODATE, &sh->dev[i].flags); ++ if (test_bit(R5_ReadError, &sh->dev[i].flags)) { ++ /* Note that this cannot happen on a ++ * replacement device. We just fail those on ++ * any error ++ */ ++ printk_ratelimited( ++ KERN_INFO ++ "md/raid:%s: read error corrected" ++ " (%lu sectors at %llu on %s)\n", ++ mdname(conf->mddev), STRIPE_SECTORS, ++ (unsigned long long)s, ++ bdevname(rdev->bdev, b)); ++ atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); ++ clear_bit(R5_ReadError, &sh->dev[i].flags); ++ clear_bit(R5_ReWrite, &sh->dev[i].flags); ++ } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) ++ clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); ++ ++ if (atomic_read(&rdev->read_errors)) ++ atomic_set(&rdev->read_errors, 0); ++ } else { ++ const char *bdn = bdevname(rdev->bdev, b); ++ int retry = 0; ++ int set_bad = 0; ++ ++ clear_bit(R5_UPTODATE, &sh->dev[i].flags); ++ atomic_inc(&rdev->read_errors); ++ if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) ++ printk_ratelimited( ++ KERN_WARNING ++ "md/raid:%s: read error on replacement device " ++ "(sector %llu on %s).\n", ++ mdname(conf->mddev), ++ (unsigned long long)s, ++ bdn); ++ else if (conf->mddev->degraded >= conf->max_degraded) { ++ set_bad = 1; ++ printk_ratelimited( ++ KERN_WARNING ++ "md/raid:%s: read error not correctable " ++ "(sector %llu on %s).\n", ++ mdname(conf->mddev), ++ (unsigned long long)s, ++ bdn); ++ } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { ++ /* Oh, no!!! */ ++ set_bad = 1; ++ printk_ratelimited( ++ KERN_WARNING ++ "md/raid:%s: read error NOT corrected!! " ++ "(sector %llu on %s).\n", ++ mdname(conf->mddev), ++ (unsigned long long)s, ++ bdn); ++ } else if (atomic_read(&rdev->read_errors) ++ > conf->max_nr_stripes) ++ printk(KERN_WARNING ++ "md/raid:%s: Too many read errors, failing device %s.\n", ++ mdname(conf->mddev), bdn); ++ else ++ retry = 1; ++ if (set_bad && test_bit(In_sync, &rdev->flags) ++ && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) ++ retry = 1; ++ if (retry) ++ if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { ++ set_bit(R5_ReadError, &sh->dev[i].flags); ++ clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); ++ } else ++ set_bit(R5_ReadNoMerge, &sh->dev[i].flags); ++ else { ++ clear_bit(R5_ReadError, &sh->dev[i].flags); ++ clear_bit(R5_ReWrite, &sh->dev[i].flags); ++ if (!(set_bad ++ && test_bit(In_sync, &rdev->flags) ++ && rdev_set_badblocks( ++ rdev, sh->sector, STRIPE_SECTORS, 0))) ++ md_error(conf->mddev, rdev); ++ } ++ } ++ rdev_dec_pending(rdev, conf->mddev); ++ clear_bit(R5_LOCKED, &sh->dev[i].flags); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++} ++ ++static void raid5_end_write_request(struct bio *bi, int error) ++{ ++ struct stripe_head *sh = bi->bi_private; ++ struct r5conf *conf = sh->raid_conf; ++ int disks = sh->disks, i; ++ struct md_rdev *uninitialized_var(rdev); ++ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); ++ sector_t first_bad; ++ int bad_sectors; ++ int replacement = 0; ++ ++ for (i = 0 ; i < disks; i++) { ++ if (bi == &sh->dev[i].req) { ++ rdev = conf->disks[i].rdev; ++ break; ++ } ++ if (bi == &sh->dev[i].rreq) { ++ rdev = conf->disks[i].replacement; ++ if (rdev) ++ replacement = 1; ++ else ++ /* rdev was removed and 'replacement' ++ * replaced it. rdev is not removed ++ * until all requests are finished. ++ */ ++ rdev = conf->disks[i].rdev; ++ break; ++ } ++ } ++ pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", ++ (unsigned long long)sh->sector, i, atomic_read(&sh->count), ++ uptodate); ++ if (i == disks) { ++ BUG(); ++ return; ++ } ++ ++ if (replacement) { ++ if (!uptodate) ++ md_error(conf->mddev, rdev); ++ else if (is_badblock(rdev, sh->sector, ++ STRIPE_SECTORS, ++ &first_bad, &bad_sectors)) ++ set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); ++ } else { ++ if (!uptodate) { ++ set_bit(STRIPE_DEGRADED, &sh->state); ++ set_bit(WriteErrorSeen, &rdev->flags); ++ set_bit(R5_WriteError, &sh->dev[i].flags); ++ if (!test_and_set_bit(WantReplacement, &rdev->flags)) ++ set_bit(MD_RECOVERY_NEEDED, ++ &rdev->mddev->recovery); ++ } else if (is_badblock(rdev, sh->sector, ++ STRIPE_SECTORS, ++ &first_bad, &bad_sectors)) { ++ set_bit(R5_MadeGood, &sh->dev[i].flags); ++ if (test_bit(R5_ReadError, &sh->dev[i].flags)) ++ /* That was a successful write so make ++ * sure it looks like we already did ++ * a re-write. ++ */ ++ set_bit(R5_ReWrite, &sh->dev[i].flags); ++ } ++ } ++ rdev_dec_pending(rdev, conf->mddev); ++ ++ if (sh->batch_head && !uptodate && !replacement) ++ set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state); ++ ++ if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) ++ clear_bit(R5_LOCKED, &sh->dev[i].flags); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++ ++ if (sh->batch_head && sh != sh->batch_head) ++ release_stripe(sh->batch_head); ++} ++ ++static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); ++ ++static void raid5_build_block(struct stripe_head *sh, int i, int previous) ++{ ++ struct r5dev *dev = &sh->dev[i]; ++ ++ bio_init(&dev->req); ++ dev->req.bi_io_vec = &dev->vec; ++ dev->req.bi_max_vecs = 1; ++ dev->req.bi_private = sh; ++ ++ bio_init(&dev->rreq); ++ dev->rreq.bi_io_vec = &dev->rvec; ++ dev->rreq.bi_max_vecs = 1; ++ dev->rreq.bi_private = sh; ++ ++ dev->flags = 0; ++ dev->sector = compute_blocknr(sh, i, previous); ++} ++ ++static void error(struct mddev *mddev, struct md_rdev *rdev) ++{ ++ char b[BDEVNAME_SIZE]; ++ struct r5conf *conf = mddev->private; ++ unsigned long flags; ++ pr_debug("raid456: error called\n"); ++ ++ spin_lock_irqsave(&conf->device_lock, flags); ++ clear_bit(In_sync, &rdev->flags); ++ mddev->degraded = calc_degraded(conf); ++ spin_unlock_irqrestore(&conf->device_lock, flags); ++ set_bit(MD_RECOVERY_INTR, &mddev->recovery); ++ ++ set_bit(Blocked, &rdev->flags); ++ set_bit(Faulty, &rdev->flags); ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ printk(KERN_ALERT ++ "md/raid:%s: Disk failure on %s, disabling device.\n" ++ "md/raid:%s: Operation continuing on %d devices.\n", ++ mdname(mddev), ++ bdevname(rdev->bdev, b), ++ mdname(mddev), ++ conf->raid_disks - mddev->degraded); ++} ++ ++/* ++ * Input: a 'big' sector number, ++ * Output: index of the data and parity disk, and the sector # in them. ++ */ ++static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, ++ int previous, int *dd_idx, ++ struct stripe_head *sh) ++{ ++ sector_t stripe, stripe2; ++ sector_t chunk_number; ++ unsigned int chunk_offset; ++ int pd_idx, qd_idx; ++ int ddf_layout = 0; ++ sector_t new_sector; ++ int algorithm = previous ? conf->prev_algo ++ : conf->algorithm; ++ int sectors_per_chunk = previous ? conf->prev_chunk_sectors ++ : conf->chunk_sectors; ++ int raid_disks = previous ? conf->previous_raid_disks ++ : conf->raid_disks; ++ int data_disks = raid_disks - conf->max_degraded; ++ ++ /* First compute the information on this sector */ ++ ++ /* ++ * Compute the chunk number and the sector offset inside the chunk ++ */ ++ chunk_offset = sector_div(r_sector, sectors_per_chunk); ++ chunk_number = r_sector; ++ ++ /* ++ * Compute the stripe number ++ */ ++ stripe = chunk_number; ++ *dd_idx = sector_div(stripe, data_disks); ++ stripe2 = stripe; ++ /* ++ * Select the parity disk based on the user selected algorithm. ++ */ ++ pd_idx = qd_idx = -1; ++ switch(conf->level) { ++ case 4: ++ pd_idx = data_disks; ++ break; ++ case 5: ++ switch (algorithm) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ pd_idx = data_disks - sector_div(stripe2, raid_disks); ++ if (*dd_idx >= pd_idx) ++ (*dd_idx)++; ++ break; ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ pd_idx = sector_div(stripe2, raid_disks); ++ if (*dd_idx >= pd_idx) ++ (*dd_idx)++; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ pd_idx = data_disks - sector_div(stripe2, raid_disks); ++ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; ++ break; ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ pd_idx = sector_div(stripe2, raid_disks); ++ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; ++ break; ++ case ALGORITHM_PARITY_0: ++ pd_idx = 0; ++ (*dd_idx)++; ++ break; ++ case ALGORITHM_PARITY_N: ++ pd_idx = data_disks; ++ break; ++ default: ++ BUG(); ++ } ++ break; ++ case 6: ++ ++ switch (algorithm) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); ++ qd_idx = pd_idx + 1; ++ if (pd_idx == raid_disks-1) { ++ (*dd_idx)++; /* Q D D D P */ ++ qd_idx = 0; ++ } else if (*dd_idx >= pd_idx) ++ (*dd_idx) += 2; /* D D P Q D */ ++ break; ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ pd_idx = sector_div(stripe2, raid_disks); ++ qd_idx = pd_idx + 1; ++ if (pd_idx == raid_disks-1) { ++ (*dd_idx)++; /* Q D D D P */ ++ qd_idx = 0; ++ } else if (*dd_idx >= pd_idx) ++ (*dd_idx) += 2; /* D D P Q D */ ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); ++ qd_idx = (pd_idx + 1) % raid_disks; ++ *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; ++ break; ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ pd_idx = sector_div(stripe2, raid_disks); ++ qd_idx = (pd_idx + 1) % raid_disks; ++ *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; ++ break; ++ ++ case ALGORITHM_PARITY_0: ++ pd_idx = 0; ++ qd_idx = 1; ++ (*dd_idx) += 2; ++ break; ++ case ALGORITHM_PARITY_N: ++ pd_idx = data_disks; ++ qd_idx = data_disks + 1; ++ break; ++ ++ case ALGORITHM_ROTATING_ZERO_RESTART: ++ /* Exactly the same as RIGHT_ASYMMETRIC, but or ++ * of blocks for computing Q is different. ++ */ ++ pd_idx = sector_div(stripe2, raid_disks); ++ qd_idx = pd_idx + 1; ++ if (pd_idx == raid_disks-1) { ++ (*dd_idx)++; /* Q D D D P */ ++ qd_idx = 0; ++ } else if (*dd_idx >= pd_idx) ++ (*dd_idx) += 2; /* D D P Q D */ ++ ddf_layout = 1; ++ break; ++ ++ case ALGORITHM_ROTATING_N_RESTART: ++ /* Same a left_asymmetric, by first stripe is ++ * D D D P Q rather than ++ * Q D D D P ++ */ ++ stripe2 += 1; ++ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); ++ qd_idx = pd_idx + 1; ++ if (pd_idx == raid_disks-1) { ++ (*dd_idx)++; /* Q D D D P */ ++ qd_idx = 0; ++ } else if (*dd_idx >= pd_idx) ++ (*dd_idx) += 2; /* D D P Q D */ ++ ddf_layout = 1; ++ break; ++ ++ case ALGORITHM_ROTATING_N_CONTINUE: ++ /* Same as left_symmetric but Q is before P */ ++ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); ++ qd_idx = (pd_idx + raid_disks - 1) % raid_disks; ++ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; ++ ddf_layout = 1; ++ break; ++ ++ case ALGORITHM_LEFT_ASYMMETRIC_6: ++ /* RAID5 left_asymmetric, with Q on last device */ ++ pd_idx = data_disks - sector_div(stripe2, raid_disks-1); ++ if (*dd_idx >= pd_idx) ++ (*dd_idx)++; ++ qd_idx = raid_disks - 1; ++ break; ++ ++ case ALGORITHM_RIGHT_ASYMMETRIC_6: ++ pd_idx = sector_div(stripe2, raid_disks-1); ++ if (*dd_idx >= pd_idx) ++ (*dd_idx)++; ++ qd_idx = raid_disks - 1; ++ break; ++ ++ case ALGORITHM_LEFT_SYMMETRIC_6: ++ pd_idx = data_disks - sector_div(stripe2, raid_disks-1); ++ *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); ++ qd_idx = raid_disks - 1; ++ break; ++ ++ case ALGORITHM_RIGHT_SYMMETRIC_6: ++ pd_idx = sector_div(stripe2, raid_disks-1); ++ *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); ++ qd_idx = raid_disks - 1; ++ break; ++ ++ case ALGORITHM_PARITY_0_6: ++ pd_idx = 0; ++ (*dd_idx)++; ++ qd_idx = raid_disks - 1; ++ break; ++ ++ default: ++ BUG(); ++ } ++ break; ++ } ++ ++ if (sh) { ++ sh->pd_idx = pd_idx; ++ sh->qd_idx = qd_idx; ++ sh->ddf_layout = ddf_layout; ++ } ++ /* ++ * Finally, compute the new sector number ++ */ ++ new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; ++ return new_sector; ++} ++ ++static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ int raid_disks = sh->disks; ++ int data_disks = raid_disks - conf->max_degraded; ++ sector_t new_sector = sh->sector, check; ++ int sectors_per_chunk = previous ? conf->prev_chunk_sectors ++ : conf->chunk_sectors; ++ int algorithm = previous ? conf->prev_algo ++ : conf->algorithm; ++ sector_t stripe; ++ int chunk_offset; ++ sector_t chunk_number; ++ int dummy1, dd_idx = i; ++ sector_t r_sector; ++ struct stripe_head sh2; ++ ++ chunk_offset = sector_div(new_sector, sectors_per_chunk); ++ stripe = new_sector; ++ ++ if (i == sh->pd_idx) ++ return 0; ++ switch(conf->level) { ++ case 4: break; ++ case 5: ++ switch (algorithm) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ if (i > sh->pd_idx) ++ i--; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ if (i < sh->pd_idx) ++ i += raid_disks; ++ i -= (sh->pd_idx + 1); ++ break; ++ case ALGORITHM_PARITY_0: ++ i -= 1; ++ break; ++ case ALGORITHM_PARITY_N: ++ break; ++ default: ++ BUG(); ++ } ++ break; ++ case 6: ++ if (i == sh->qd_idx) ++ return 0; /* It is the Q disk */ ++ switch (algorithm) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ case ALGORITHM_ROTATING_ZERO_RESTART: ++ case ALGORITHM_ROTATING_N_RESTART: ++ if (sh->pd_idx == raid_disks-1) ++ i--; /* Q D D D P */ ++ else if (i > sh->pd_idx) ++ i -= 2; /* D D P Q D */ ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ if (sh->pd_idx == raid_disks-1) ++ i--; /* Q D D D P */ ++ else { ++ /* D D P Q D */ ++ if (i < sh->pd_idx) ++ i += raid_disks; ++ i -= (sh->pd_idx + 2); ++ } ++ break; ++ case ALGORITHM_PARITY_0: ++ i -= 2; ++ break; ++ case ALGORITHM_PARITY_N: ++ break; ++ case ALGORITHM_ROTATING_N_CONTINUE: ++ /* Like left_symmetric, but P is before Q */ ++ if (sh->pd_idx == 0) ++ i--; /* P D D D Q */ ++ else { ++ /* D D Q P D */ ++ if (i < sh->pd_idx) ++ i += raid_disks; ++ i -= (sh->pd_idx + 1); ++ } ++ break; ++ case ALGORITHM_LEFT_ASYMMETRIC_6: ++ case ALGORITHM_RIGHT_ASYMMETRIC_6: ++ if (i > sh->pd_idx) ++ i--; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC_6: ++ case ALGORITHM_RIGHT_SYMMETRIC_6: ++ if (i < sh->pd_idx) ++ i += data_disks + 1; ++ i -= (sh->pd_idx + 1); ++ break; ++ case ALGORITHM_PARITY_0_6: ++ i -= 1; ++ break; ++ default: ++ BUG(); ++ } ++ break; ++ } ++ ++ chunk_number = stripe * data_disks + i; ++ r_sector = chunk_number * sectors_per_chunk + chunk_offset; ++ ++ check = raid5_compute_sector(conf, r_sector, ++ previous, &dummy1, &sh2); ++ if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx ++ || sh2.qd_idx != sh->qd_idx) { ++ printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", ++ mdname(conf->mddev)); ++ return 0; ++ } ++ return r_sector; ++} ++ ++static void ++schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, ++ int rcw, int expand) ++{ ++ int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks; ++ struct r5conf *conf = sh->raid_conf; ++ int level = conf->level; ++ ++ if (rcw) { ++ ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ ++ if (dev->towrite) { ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantdrain, &dev->flags); ++ if (!expand) ++ clear_bit(R5_UPTODATE, &dev->flags); ++ s->locked++; ++ } ++ } ++ /* if we are not expanding this is a proper write request, and ++ * there will be bios with new data to be drained into the ++ * stripe cache ++ */ ++ if (!expand) { ++ if (!s->locked) ++ /* False alarm, nothing to do */ ++ return; ++ sh->reconstruct_state = reconstruct_state_drain_run; ++ set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); ++ } else ++ sh->reconstruct_state = reconstruct_state_run; ++ ++ set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); ++ ++ if (s->locked + conf->max_degraded == disks) ++ if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) ++ atomic_inc(&conf->pending_full_writes); ++ } else { ++ BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || ++ test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); ++ BUG_ON(level == 6 && ++ (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) || ++ test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags)))); ++ ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (i == pd_idx || i == qd_idx) ++ continue; ++ ++ if (dev->towrite && ++ (test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags))) { ++ set_bit(R5_Wantdrain, &dev->flags); ++ set_bit(R5_LOCKED, &dev->flags); ++ clear_bit(R5_UPTODATE, &dev->flags); ++ s->locked++; ++ } ++ } ++ if (!s->locked) ++ /* False alarm - nothing to do */ ++ return; ++ sh->reconstruct_state = reconstruct_state_prexor_drain_run; ++ set_bit(STRIPE_OP_PREXOR, &s->ops_request); ++ set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); ++ set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); ++ } ++ ++ /* keep the parity disk(s) locked while asynchronous operations ++ * are in flight ++ */ ++ set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); ++ clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); ++ s->locked++; ++ ++ if (level == 6) { ++ int qd_idx = sh->qd_idx; ++ struct r5dev *dev = &sh->dev[qd_idx]; ++ ++ set_bit(R5_LOCKED, &dev->flags); ++ clear_bit(R5_UPTODATE, &dev->flags); ++ s->locked++; ++ } ++ ++ pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", ++ __func__, (unsigned long long)sh->sector, ++ s->locked, s->ops_request); ++} ++ ++/* ++ * Each stripe/dev can have one or more bion attached. ++ * toread/towrite point to the first in a chain. ++ * The bi_next chain must be in order. ++ */ ++static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, ++ int forwrite, int previous) ++{ ++ struct bio **bip; ++ struct r5conf *conf = sh->raid_conf; ++ int firstwrite=0; ++ ++ pr_debug("adding bi b#%llu to stripe s#%llu\n", ++ (unsigned long long)bi->bi_iter.bi_sector, ++ (unsigned long long)sh->sector); ++ ++ /* ++ * If several bio share a stripe. The bio bi_phys_segments acts as a ++ * reference count to avoid race. The reference count should already be ++ * increased before this function is called (for example, in ++ * make_request()), so other bio sharing this stripe will not free the ++ * stripe. If a stripe is owned by one stripe, the stripe lock will ++ * protect it. ++ */ ++ spin_lock_irq(&sh->stripe_lock); ++ /* Don't allow new IO added to stripes in batch list */ ++ if (sh->batch_head) ++ goto overlap; ++ if (forwrite) { ++ bip = &sh->dev[dd_idx].towrite; ++ if (*bip == NULL) ++ firstwrite = 1; ++ } else ++ bip = &sh->dev[dd_idx].toread; ++ while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) { ++ if (bio_end_sector(*bip) > bi->bi_iter.bi_sector) ++ goto overlap; ++ bip = & (*bip)->bi_next; ++ } ++ if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi)) ++ goto overlap; ++ ++ if (!forwrite || previous) ++ clear_bit(STRIPE_BATCH_READY, &sh->state); ++ ++ BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); ++ if (*bip) ++ bi->bi_next = *bip; ++ *bip = bi; ++ raid5_inc_bi_active_stripes(bi); ++ ++ if (forwrite) { ++ /* check if page is covered */ ++ sector_t sector = sh->dev[dd_idx].sector; ++ for (bi=sh->dev[dd_idx].towrite; ++ sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && ++ bi && bi->bi_iter.bi_sector <= sector; ++ bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { ++ if (bio_end_sector(bi) >= sector) ++ sector = bio_end_sector(bi); ++ } ++ if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) ++ if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags)) ++ sh->overwrite_disks++; ++ } ++ ++ pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", ++ (unsigned long long)(*bip)->bi_iter.bi_sector, ++ (unsigned long long)sh->sector, dd_idx); ++ ++ if (conf->mddev->bitmap && firstwrite) { ++ /* Cannot hold spinlock over bitmap_startwrite, ++ * but must ensure this isn't added to a batch until ++ * we have added to the bitmap and set bm_seq. ++ * So set STRIPE_BITMAP_PENDING to prevent ++ * batching. ++ * If multiple add_stripe_bio() calls race here they ++ * much all set STRIPE_BITMAP_PENDING. So only the first one ++ * to complete "bitmap_startwrite" gets to set ++ * STRIPE_BIT_DELAY. This is important as once a stripe ++ * is added to a batch, STRIPE_BIT_DELAY cannot be changed ++ * any more. ++ */ ++ set_bit(STRIPE_BITMAP_PENDING, &sh->state); ++ spin_unlock_irq(&sh->stripe_lock); ++ bitmap_startwrite(conf->mddev->bitmap, sh->sector, ++ STRIPE_SECTORS, 0); ++ spin_lock_irq(&sh->stripe_lock); ++ clear_bit(STRIPE_BITMAP_PENDING, &sh->state); ++ if (!sh->batch_head) { ++ sh->bm_seq = conf->seq_flush+1; ++ set_bit(STRIPE_BIT_DELAY, &sh->state); ++ } ++ } ++ spin_unlock_irq(&sh->stripe_lock); ++ ++ if (stripe_can_batch(sh)) ++ stripe_add_to_batch_list(conf, sh); ++ return 1; ++ ++ overlap: ++ set_bit(R5_Overlap, &sh->dev[dd_idx].flags); ++ spin_unlock_irq(&sh->stripe_lock); ++ return 0; ++} ++ ++static void end_reshape(struct r5conf *conf); ++ ++static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, ++ struct stripe_head *sh) ++{ ++ int sectors_per_chunk = ++ previous ? conf->prev_chunk_sectors : conf->chunk_sectors; ++ int dd_idx; ++ int chunk_offset = sector_div(stripe, sectors_per_chunk); ++ int disks = previous ? conf->previous_raid_disks : conf->raid_disks; ++ ++ raid5_compute_sector(conf, ++ stripe * (disks - conf->max_degraded) ++ *sectors_per_chunk + chunk_offset, ++ previous, ++ &dd_idx, sh); ++} ++ ++static void ++handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, ++ struct stripe_head_state *s, int disks, ++ struct bio **return_bi) ++{ ++ int i; ++ BUG_ON(sh->batch_head); ++ for (i = disks; i--; ) { ++ struct bio *bi; ++ int bitmap_end = 0; ++ ++ if (test_bit(R5_ReadError, &sh->dev[i].flags)) { ++ struct md_rdev *rdev; ++ rcu_read_lock(); ++ rdev = rcu_dereference(conf->disks[i].rdev); ++ if (rdev && test_bit(In_sync, &rdev->flags)) ++ atomic_inc(&rdev->nr_pending); ++ else ++ rdev = NULL; ++ rcu_read_unlock(); ++ if (rdev) { ++ if (!rdev_set_badblocks( ++ rdev, ++ sh->sector, ++ STRIPE_SECTORS, 0)) ++ md_error(conf->mddev, rdev); ++ rdev_dec_pending(rdev, conf->mddev); ++ } ++ } ++ spin_lock_irq(&sh->stripe_lock); ++ /* fail all writes first */ ++ bi = sh->dev[i].towrite; ++ sh->dev[i].towrite = NULL; ++ sh->overwrite_disks = 0; ++ spin_unlock_irq(&sh->stripe_lock); ++ if (bi) ++ bitmap_end = 1; ++ ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) ++ wake_up(&conf->wait_for_overlap); ++ ++ while (bi && bi->bi_iter.bi_sector < ++ sh->dev[i].sector + STRIPE_SECTORS) { ++ struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); ++ clear_bit(BIO_UPTODATE, &bi->bi_flags); ++ if (!raid5_dec_bi_active_stripes(bi)) { ++ md_write_end(conf->mddev); ++ bi->bi_next = *return_bi; ++ *return_bi = bi; ++ } ++ bi = nextbi; ++ } ++ if (bitmap_end) ++ bitmap_endwrite(conf->mddev->bitmap, sh->sector, ++ STRIPE_SECTORS, 0, 0); ++ bitmap_end = 0; ++ /* and fail all 'written' */ ++ bi = sh->dev[i].written; ++ sh->dev[i].written = NULL; ++ if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) { ++ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); ++ sh->dev[i].page = sh->dev[i].orig_page; ++ } ++ ++ if (bi) bitmap_end = 1; ++ while (bi && bi->bi_iter.bi_sector < ++ sh->dev[i].sector + STRIPE_SECTORS) { ++ struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); ++ clear_bit(BIO_UPTODATE, &bi->bi_flags); ++ if (!raid5_dec_bi_active_stripes(bi)) { ++ md_write_end(conf->mddev); ++ bi->bi_next = *return_bi; ++ *return_bi = bi; ++ } ++ bi = bi2; ++ } ++ ++ /* fail any reads if this device is non-operational and ++ * the data has not reached the cache yet. ++ */ ++ if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && ++ (!test_bit(R5_Insync, &sh->dev[i].flags) || ++ test_bit(R5_ReadError, &sh->dev[i].flags))) { ++ spin_lock_irq(&sh->stripe_lock); ++ bi = sh->dev[i].toread; ++ sh->dev[i].toread = NULL; ++ spin_unlock_irq(&sh->stripe_lock); ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) ++ wake_up(&conf->wait_for_overlap); ++ while (bi && bi->bi_iter.bi_sector < ++ sh->dev[i].sector + STRIPE_SECTORS) { ++ struct bio *nextbi = ++ r5_next_bio(bi, sh->dev[i].sector); ++ clear_bit(BIO_UPTODATE, &bi->bi_flags); ++ if (!raid5_dec_bi_active_stripes(bi)) { ++ bi->bi_next = *return_bi; ++ *return_bi = bi; ++ } ++ bi = nextbi; ++ } ++ } ++ if (bitmap_end) ++ bitmap_endwrite(conf->mddev->bitmap, sh->sector, ++ STRIPE_SECTORS, 0, 0); ++ /* If we were in the middle of a write the parity block might ++ * still be locked - so just clear all R5_LOCKED flags ++ */ ++ clear_bit(R5_LOCKED, &sh->dev[i].flags); ++ } ++ ++ if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) ++ if (atomic_dec_and_test(&conf->pending_full_writes)) ++ md_wakeup_thread(conf->mddev->thread); ++} ++ ++static void ++handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, ++ struct stripe_head_state *s) ++{ ++ int abort = 0; ++ int i; ++ ++ BUG_ON(sh->batch_head); ++ clear_bit(STRIPE_SYNCING, &sh->state); ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) ++ wake_up(&conf->wait_for_overlap); ++ s->syncing = 0; ++ s->replacing = 0; ++ /* There is nothing more to do for sync/check/repair. ++ * Don't even need to abort as that is handled elsewhere ++ * if needed, and not always wanted e.g. if there is a known ++ * bad block here. ++ * For recover/replace we need to record a bad block on all ++ * non-sync devices, or abort the recovery ++ */ ++ if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { ++ /* During recovery devices cannot be removed, so ++ * locking and refcounting of rdevs is not needed ++ */ ++ for (i = 0; i < conf->raid_disks; i++) { ++ struct md_rdev *rdev = conf->disks[i].rdev; ++ if (rdev ++ && !test_bit(Faulty, &rdev->flags) ++ && !test_bit(In_sync, &rdev->flags) ++ && !rdev_set_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0)) ++ abort = 1; ++ rdev = conf->disks[i].replacement; ++ if (rdev ++ && !test_bit(Faulty, &rdev->flags) ++ && !test_bit(In_sync, &rdev->flags) ++ && !rdev_set_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0)) ++ abort = 1; ++ } ++ if (abort) ++ conf->recovery_disabled = ++ conf->mddev->recovery_disabled; ++ } ++ md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); ++} ++ ++static int want_replace(struct stripe_head *sh, int disk_idx) ++{ ++ struct md_rdev *rdev; ++ int rv = 0; ++ /* Doing recovery so rcu locking not required */ ++ rdev = sh->raid_conf->disks[disk_idx].replacement; ++ if (rdev ++ && !test_bit(Faulty, &rdev->flags) ++ && !test_bit(In_sync, &rdev->flags) ++ && (rdev->recovery_offset <= sh->sector ++ || rdev->mddev->recovery_cp <= sh->sector)) ++ rv = 1; ++ ++ return rv; ++} ++ ++/* fetch_block - checks the given member device to see if its data needs ++ * to be read or computed to satisfy a request. ++ * ++ * Returns 1 when no more member devices need to be checked, otherwise returns ++ * 0 to tell the loop in handle_stripe_fill to continue ++ */ ++ ++static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, ++ int disk_idx, int disks) ++{ ++ struct r5dev *dev = &sh->dev[disk_idx]; ++ struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], ++ &sh->dev[s->failed_num[1]] }; ++ int i; ++ ++ ++ if (test_bit(R5_LOCKED, &dev->flags) || ++ test_bit(R5_UPTODATE, &dev->flags)) ++ /* No point reading this as we already have it or have ++ * decided to get it. ++ */ ++ return 0; ++ ++ if (dev->toread || ++ (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags))) ++ /* We need this block to directly satisfy a request */ ++ return 1; ++ ++ if (s->syncing || s->expanding || ++ (s->replacing && want_replace(sh, disk_idx))) ++ /* When syncing, or expanding we read everything. ++ * When replacing, we need the replaced block. ++ */ ++ return 1; ++ ++ if ((s->failed >= 1 && fdev[0]->toread) || ++ (s->failed >= 2 && fdev[1]->toread)) ++ /* If we want to read from a failed device, then ++ * we need to actually read every other device. ++ */ ++ return 1; ++ ++ /* Sometimes neither read-modify-write nor reconstruct-write ++ * cycles can work. In those cases we read every block we ++ * can. Then the parity-update is certain to have enough to ++ * work with. ++ * This can only be a problem when we need to write something, ++ * and some device has failed. If either of those tests ++ * fail we need look no further. ++ */ ++ if (!s->failed || !s->to_write) ++ return 0; ++ ++ if (test_bit(R5_Insync, &dev->flags) && ++ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ /* Pre-reads at not permitted until after short delay ++ * to gather multiple requests. However if this ++ * device is no Insync, the block could only be be computed ++ * and there is no need to delay that. ++ */ ++ return 0; ++ ++ for (i = 0; i < s->failed; i++) { ++ if (fdev[i]->towrite && ++ !test_bit(R5_UPTODATE, &fdev[i]->flags) && ++ !test_bit(R5_OVERWRITE, &fdev[i]->flags)) ++ /* If we have a partial write to a failed ++ * device, then we will need to reconstruct ++ * the content of that device, so all other ++ * devices must be read. ++ */ ++ return 1; ++ } ++ ++ /* If we are forced to do a reconstruct-write, either because ++ * the current RAID6 implementation only supports that, or ++ * or because parity cannot be trusted and we are currently ++ * recovering it, there is extra need to be careful. ++ * If one of the devices that we would need to read, because ++ * it is not being overwritten (and maybe not written at all) ++ * is missing/faulty, then we need to read everything we can. ++ */ ++ if (sh->raid_conf->level != 6 && ++ sh->sector < sh->raid_conf->mddev->recovery_cp) ++ /* reconstruct-write isn't being forced */ ++ return 0; ++ for (i = 0; i < s->failed; i++) { ++ if (s->failed_num[i] != sh->pd_idx && ++ s->failed_num[i] != sh->qd_idx && ++ !test_bit(R5_UPTODATE, &fdev[i]->flags) && ++ !test_bit(R5_OVERWRITE, &fdev[i]->flags)) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, ++ int disk_idx, int disks) ++{ ++ struct r5dev *dev = &sh->dev[disk_idx]; ++ ++ /* is the data in this block needed, and can we get it? */ ++ if (need_this_block(sh, s, disk_idx, disks)) { ++ /* we would like to get this block, possibly by computing it, ++ * otherwise read it if the backing disk is insync ++ */ ++ BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); ++ BUG_ON(test_bit(R5_Wantread, &dev->flags)); ++ BUG_ON(sh->batch_head); ++ if ((s->uptodate == disks - 1) && ++ (s->failed && (disk_idx == s->failed_num[0] || ++ disk_idx == s->failed_num[1]))) { ++ /* have disk failed, and we're requested to fetch it; ++ * do compute it ++ */ ++ pr_debug("Computing stripe %llu block %d\n", ++ (unsigned long long)sh->sector, disk_idx); ++ set_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); ++ set_bit(R5_Wantcompute, &dev->flags); ++ sh->ops.target = disk_idx; ++ sh->ops.target2 = -1; /* no 2nd target */ ++ s->req_compute = 1; ++ /* Careful: from this point on 'uptodate' is in the eye ++ * of raid_run_ops which services 'compute' operations ++ * before writes. R5_Wantcompute flags a block that will ++ * be R5_UPTODATE by the time it is needed for a ++ * subsequent operation. ++ */ ++ s->uptodate++; ++ return 1; ++ } else if (s->uptodate == disks-2 && s->failed >= 2) { ++ /* Computing 2-failure is *very* expensive; only ++ * do it if failed >= 2 ++ */ ++ int other; ++ for (other = disks; other--; ) { ++ if (other == disk_idx) ++ continue; ++ if (!test_bit(R5_UPTODATE, ++ &sh->dev[other].flags)) ++ break; ++ } ++ BUG_ON(other < 0); ++ pr_debug("Computing stripe %llu blocks %d,%d\n", ++ (unsigned long long)sh->sector, ++ disk_idx, other); ++ set_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); ++ set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); ++ set_bit(R5_Wantcompute, &sh->dev[other].flags); ++ sh->ops.target = disk_idx; ++ sh->ops.target2 = other; ++ s->uptodate += 2; ++ s->req_compute = 1; ++ return 1; ++ } else if (test_bit(R5_Insync, &dev->flags)) { ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantread, &dev->flags); ++ s->locked++; ++ pr_debug("Reading block %d (sync=%d)\n", ++ disk_idx, s->syncing); ++ } ++ } ++ ++ return 0; ++} ++ ++/** ++ * handle_stripe_fill - read or compute data to satisfy pending requests. ++ */ ++static void handle_stripe_fill(struct stripe_head *sh, ++ struct stripe_head_state *s, ++ int disks) ++{ ++ int i; ++ ++ /* look for blocks to read/compute, skip this if a compute ++ * is already in flight, or if the stripe contents are in the ++ * midst of changing due to a write ++ */ ++ if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && ++ !sh->reconstruct_state) ++ for (i = disks; i--; ) ++ if (fetch_block(sh, s, i, disks)) ++ break; ++ set_bit(STRIPE_HANDLE, &sh->state); ++} ++ ++static void break_stripe_batch_list(struct stripe_head *head_sh, ++ unsigned long handle_flags); ++/* handle_stripe_clean_event ++ * any written block on an uptodate or failed drive can be returned. ++ * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but ++ * never LOCKED, so we don't need to test 'failed' directly. ++ */ ++static void handle_stripe_clean_event(struct r5conf *conf, ++ struct stripe_head *sh, int disks, struct bio **return_bi) ++{ ++ int i; ++ struct r5dev *dev; ++ int discard_pending = 0; ++ struct stripe_head *head_sh = sh; ++ bool do_endio = false; ++ ++ for (i = disks; i--; ) ++ if (sh->dev[i].written) { ++ dev = &sh->dev[i]; ++ if (!test_bit(R5_LOCKED, &dev->flags) && ++ (test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Discard, &dev->flags) || ++ test_bit(R5_SkipCopy, &dev->flags))) { ++ /* We can return any write requests */ ++ struct bio *wbi, *wbi2; ++ pr_debug("Return write for disc %d\n", i); ++ if (test_and_clear_bit(R5_Discard, &dev->flags)) ++ clear_bit(R5_UPTODATE, &dev->flags); ++ if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) { ++ WARN_ON(test_bit(R5_UPTODATE, &dev->flags)); ++ } ++ do_endio = true; ++ ++returnbi: ++ dev->page = dev->orig_page; ++ wbi = dev->written; ++ dev->written = NULL; ++ while (wbi && wbi->bi_iter.bi_sector < ++ dev->sector + STRIPE_SECTORS) { ++ wbi2 = r5_next_bio(wbi, dev->sector); ++ if (!raid5_dec_bi_active_stripes(wbi)) { ++ md_write_end(conf->mddev); ++ wbi->bi_next = *return_bi; ++ *return_bi = wbi; ++ } ++ wbi = wbi2; ++ } ++ bitmap_endwrite(conf->mddev->bitmap, sh->sector, ++ STRIPE_SECTORS, ++ !test_bit(STRIPE_DEGRADED, &sh->state), ++ 0); ++ if (head_sh->batch_head) { ++ sh = list_first_entry(&sh->batch_list, ++ struct stripe_head, ++ batch_list); ++ if (sh != head_sh) { ++ dev = &sh->dev[i]; ++ goto returnbi; ++ } ++ } ++ sh = head_sh; ++ dev = &sh->dev[i]; ++ } else if (test_bit(R5_Discard, &dev->flags)) ++ discard_pending = 1; ++ WARN_ON(test_bit(R5_SkipCopy, &dev->flags)); ++ WARN_ON(dev->page != dev->orig_page); ++ } ++ if (!discard_pending && ++ test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) { ++ clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); ++ clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); ++ if (sh->qd_idx >= 0) { ++ clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); ++ clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags); ++ } ++ /* now that discard is done we can proceed with any sync */ ++ clear_bit(STRIPE_DISCARD, &sh->state); ++ /* ++ * SCSI discard will change some bio fields and the stripe has ++ * no updated data, so remove it from hash list and the stripe ++ * will be reinitialized ++ */ ++ spin_lock_irq(&conf->device_lock); ++unhash: ++ remove_hash(sh); ++ if (head_sh->batch_head) { ++ sh = list_first_entry(&sh->batch_list, ++ struct stripe_head, batch_list); ++ if (sh != head_sh) ++ goto unhash; ++ } ++ spin_unlock_irq(&conf->device_lock); ++ sh = head_sh; ++ ++ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) ++ set_bit(STRIPE_HANDLE, &sh->state); ++ ++ } ++ ++ if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) ++ if (atomic_dec_and_test(&conf->pending_full_writes)) ++ md_wakeup_thread(conf->mddev->thread); ++ ++ if (head_sh->batch_head && do_endio) ++ break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS); ++} ++ ++static void handle_stripe_dirtying(struct r5conf *conf, ++ struct stripe_head *sh, ++ struct stripe_head_state *s, ++ int disks) ++{ ++ int rmw = 0, rcw = 0, i; ++ sector_t recovery_cp = conf->mddev->recovery_cp; ++ ++ /* Check whether resync is now happening or should start. ++ * If yes, then the array is dirty (after unclean shutdown or ++ * initial creation), so parity in some stripes might be inconsistent. ++ * In this case, we need to always do reconstruct-write, to ensure ++ * that in case of drive failure or read-error correction, we ++ * generate correct data from the parity. ++ */ ++ if (conf->rmw_level == PARITY_DISABLE_RMW || ++ (recovery_cp < MaxSector && sh->sector >= recovery_cp && ++ s->failed == 0)) { ++ /* Calculate the real rcw later - for now make it ++ * look like rcw is cheaper ++ */ ++ rcw = 1; rmw = 2; ++ pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n", ++ conf->rmw_level, (unsigned long long)recovery_cp, ++ (unsigned long long)sh->sector); ++ } else for (i = disks; i--; ) { ++ /* would I have to read this buffer for read_modify_write */ ++ struct r5dev *dev = &sh->dev[i]; ++ if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) && ++ !test_bit(R5_LOCKED, &dev->flags) && ++ !(test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags))) { ++ if (test_bit(R5_Insync, &dev->flags)) ++ rmw++; ++ else ++ rmw += 2*disks; /* cannot read it */ ++ } ++ /* Would I have to read this buffer for reconstruct_write */ ++ if (!test_bit(R5_OVERWRITE, &dev->flags) && ++ i != sh->pd_idx && i != sh->qd_idx && ++ !test_bit(R5_LOCKED, &dev->flags) && ++ !(test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags))) { ++ if (test_bit(R5_Insync, &dev->flags)) ++ rcw++; ++ else ++ rcw += 2*disks; ++ } ++ } ++ pr_debug("for sector %llu, rmw=%d rcw=%d\n", ++ (unsigned long long)sh->sector, rmw, rcw); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_ENABLE_RMW)) && rmw > 0) { ++ /* prefer read-modify-write, but need to get some data */ ++ if (conf->mddev->queue) ++ blk_add_trace_msg(conf->mddev->queue, ++ "raid5 rmw %llu %d", ++ (unsigned long long)sh->sector, rmw); ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) && ++ !test_bit(R5_LOCKED, &dev->flags) && ++ !(test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags)) && ++ test_bit(R5_Insync, &dev->flags)) { ++ if (test_bit(STRIPE_PREREAD_ACTIVE, ++ &sh->state)) { ++ pr_debug("Read_old block %d for r-m-w\n", ++ i); ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantread, &dev->flags); ++ s->locked++; ++ } else { ++ set_bit(STRIPE_DELAYED, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ } ++ } ++ } ++ } ++ if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_ENABLE_RMW)) && rcw > 0) { ++ /* want reconstruct write, but need to get some data */ ++ int qread =0; ++ rcw = 0; ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (!test_bit(R5_OVERWRITE, &dev->flags) && ++ i != sh->pd_idx && i != sh->qd_idx && ++ !test_bit(R5_LOCKED, &dev->flags) && ++ !(test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags))) { ++ rcw++; ++ if (test_bit(R5_Insync, &dev->flags) && ++ test_bit(STRIPE_PREREAD_ACTIVE, ++ &sh->state)) { ++ pr_debug("Read_old block " ++ "%d for Reconstruct\n", i); ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantread, &dev->flags); ++ s->locked++; ++ qread++; ++ } else { ++ set_bit(STRIPE_DELAYED, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ } ++ } ++ } ++ if (rcw && conf->mddev->queue) ++ blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d", ++ (unsigned long long)sh->sector, ++ rcw, qread, test_bit(STRIPE_DELAYED, &sh->state)); ++ } ++ ++ if (rcw > disks && rmw > disks && ++ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ set_bit(STRIPE_DELAYED, &sh->state); ++ ++ /* now if nothing is locked, and if we have enough data, ++ * we can start a write request ++ */ ++ /* since handle_stripe can be called at any time we need to handle the ++ * case where a compute block operation has been submitted and then a ++ * subsequent call wants to start a write request. raid_run_ops only ++ * handles the case where compute block and reconstruct are requested ++ * simultaneously. If this is not the case then new writes need to be ++ * held off until the compute completes. ++ */ ++ if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && ++ (s->locked == 0 && (rcw == 0 || rmw == 0) && ++ !test_bit(STRIPE_BIT_DELAY, &sh->state))) ++ schedule_reconstruction(sh, s, rcw == 0, 0); ++} ++ ++static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, ++ struct stripe_head_state *s, int disks) ++{ ++ struct r5dev *dev = NULL; ++ ++ BUG_ON(sh->batch_head); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ ++ switch (sh->check_state) { ++ case check_state_idle: ++ /* start a new check operation if there are no failures */ ++ if (s->failed == 0) { ++ BUG_ON(s->uptodate != disks); ++ sh->check_state = check_state_run; ++ set_bit(STRIPE_OP_CHECK, &s->ops_request); ++ clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); ++ s->uptodate--; ++ break; ++ } ++ dev = &sh->dev[s->failed_num[0]]; ++ /* fall through */ ++ case check_state_compute_result: ++ sh->check_state = check_state_idle; ++ if (!dev) ++ dev = &sh->dev[sh->pd_idx]; ++ ++ /* check that a write has not made the stripe insync */ ++ if (test_bit(STRIPE_INSYNC, &sh->state)) ++ break; ++ ++ /* either failed parity check, or recovery is happening */ ++ BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); ++ BUG_ON(s->uptodate != disks); ++ ++ set_bit(R5_LOCKED, &dev->flags); ++ s->locked++; ++ set_bit(R5_Wantwrite, &dev->flags); ++ ++ clear_bit(STRIPE_DEGRADED, &sh->state); ++ set_bit(STRIPE_INSYNC, &sh->state); ++ break; ++ case check_state_run: ++ break; /* we will be called again upon completion */ ++ case check_state_check_result: ++ sh->check_state = check_state_idle; ++ ++ /* if a failure occurred during the check operation, leave ++ * STRIPE_INSYNC not set and let the stripe be handled again ++ */ ++ if (s->failed) ++ break; ++ ++ /* handle a successful check operation, if parity is correct ++ * we are done. Otherwise update the mismatch count and repair ++ * parity if !MD_RECOVERY_CHECK ++ */ ++ if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) ++ /* parity is correct (on disc, ++ * not in buffer any more) ++ */ ++ set_bit(STRIPE_INSYNC, &sh->state); ++ else { ++ atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); ++ if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) ++ /* don't try to repair!! */ ++ set_bit(STRIPE_INSYNC, &sh->state); ++ else { ++ sh->check_state = check_state_compute_run; ++ set_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); ++ set_bit(R5_Wantcompute, ++ &sh->dev[sh->pd_idx].flags); ++ sh->ops.target = sh->pd_idx; ++ sh->ops.target2 = -1; ++ s->uptodate++; ++ } ++ } ++ break; ++ case check_state_compute_run: ++ break; ++ default: ++ printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", ++ __func__, sh->check_state, ++ (unsigned long long) sh->sector); ++ BUG(); ++ } ++} ++ ++static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, ++ struct stripe_head_state *s, ++ int disks) ++{ ++ int pd_idx = sh->pd_idx; ++ int qd_idx = sh->qd_idx; ++ struct r5dev *dev; ++ ++ BUG_ON(sh->batch_head); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ ++ BUG_ON(s->failed > 2); ++ ++ /* Want to check and possibly repair P and Q. ++ * However there could be one 'failed' device, in which ++ * case we can only check one of them, possibly using the ++ * other to generate missing data ++ */ ++ ++ switch (sh->check_state) { ++ case check_state_idle: ++ /* start a new check operation if there are < 2 failures */ ++ if (s->failed == s->q_failed) { ++ /* The only possible failed device holds Q, so it ++ * makes sense to check P (If anything else were failed, ++ * we would have used P to recreate it). ++ */ ++ sh->check_state = check_state_run; ++ } ++ if (!s->q_failed && s->failed < 2) { ++ /* Q is not failed, and we didn't use it to generate ++ * anything, so it makes sense to check it ++ */ ++ if (sh->check_state == check_state_run) ++ sh->check_state = check_state_run_pq; ++ else ++ sh->check_state = check_state_run_q; ++ } ++ ++ /* discard potentially stale zero_sum_result */ ++ sh->ops.zero_sum_result = 0; ++ ++ if (sh->check_state == check_state_run) { ++ /* async_xor_zero_sum destroys the contents of P */ ++ clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); ++ s->uptodate--; ++ } ++ if (sh->check_state >= check_state_run && ++ sh->check_state <= check_state_run_pq) { ++ /* async_syndrome_zero_sum preserves P and Q, so ++ * no need to mark them !uptodate here ++ */ ++ set_bit(STRIPE_OP_CHECK, &s->ops_request); ++ break; ++ } ++ ++ /* we have 2-disk failure */ ++ BUG_ON(s->failed != 2); ++ /* fall through */ ++ case check_state_compute_result: ++ sh->check_state = check_state_idle; ++ ++ /* check that a write has not made the stripe insync */ ++ if (test_bit(STRIPE_INSYNC, &sh->state)) ++ break; ++ ++ /* now write out any block on a failed drive, ++ * or P or Q if they were recomputed ++ */ ++ BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ ++ if (s->failed == 2) { ++ dev = &sh->dev[s->failed_num[1]]; ++ s->locked++; ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantwrite, &dev->flags); ++ } ++ if (s->failed >= 1) { ++ dev = &sh->dev[s->failed_num[0]]; ++ s->locked++; ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantwrite, &dev->flags); ++ } ++ if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { ++ dev = &sh->dev[pd_idx]; ++ s->locked++; ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantwrite, &dev->flags); ++ } ++ if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { ++ dev = &sh->dev[qd_idx]; ++ s->locked++; ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantwrite, &dev->flags); ++ } ++ clear_bit(STRIPE_DEGRADED, &sh->state); ++ ++ set_bit(STRIPE_INSYNC, &sh->state); ++ break; ++ case check_state_run: ++ case check_state_run_q: ++ case check_state_run_pq: ++ break; /* we will be called again upon completion */ ++ case check_state_check_result: ++ sh->check_state = check_state_idle; ++ ++ /* handle a successful check operation, if parity is correct ++ * we are done. Otherwise update the mismatch count and repair ++ * parity if !MD_RECOVERY_CHECK ++ */ ++ if (sh->ops.zero_sum_result == 0) { ++ /* both parities are correct */ ++ if (!s->failed) ++ set_bit(STRIPE_INSYNC, &sh->state); ++ else { ++ /* in contrast to the raid5 case we can validate ++ * parity, but still have a failure to write ++ * back ++ */ ++ sh->check_state = check_state_compute_result; ++ /* Returning at this point means that we may go ++ * off and bring p and/or q uptodate again so ++ * we make sure to check zero_sum_result again ++ * to verify if p or q need writeback ++ */ ++ } ++ } else { ++ atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); ++ if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) ++ /* don't try to repair!! */ ++ set_bit(STRIPE_INSYNC, &sh->state); ++ else { ++ int *target = &sh->ops.target; ++ ++ sh->ops.target = -1; ++ sh->ops.target2 = -1; ++ sh->check_state = check_state_compute_run; ++ set_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); ++ if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { ++ set_bit(R5_Wantcompute, ++ &sh->dev[pd_idx].flags); ++ *target = pd_idx; ++ target = &sh->ops.target2; ++ s->uptodate++; ++ } ++ if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { ++ set_bit(R5_Wantcompute, ++ &sh->dev[qd_idx].flags); ++ *target = qd_idx; ++ s->uptodate++; ++ } ++ } ++ } ++ break; ++ case check_state_compute_run: ++ break; ++ default: ++ printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", ++ __func__, sh->check_state, ++ (unsigned long long) sh->sector); ++ BUG(); ++ } ++} ++ ++static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) ++{ ++ int i; ++ ++ /* We have read all the blocks in this stripe and now we need to ++ * copy some of them into a target stripe for expand. ++ */ ++ struct dma_async_tx_descriptor *tx = NULL; ++ BUG_ON(sh->batch_head); ++ clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); ++ for (i = 0; i < sh->disks; i++) ++ if (i != sh->pd_idx && i != sh->qd_idx) { ++ int dd_idx, j; ++ struct stripe_head *sh2; ++ struct async_submit_ctl submit; ++ ++ sector_t bn = compute_blocknr(sh, i, 1); ++ sector_t s = raid5_compute_sector(conf, bn, 0, ++ &dd_idx, NULL); ++ sh2 = get_active_stripe(conf, s, 0, 1, 1); ++ if (sh2 == NULL) ++ /* so far only the early blocks of this stripe ++ * have been requested. When later blocks ++ * get requested, we will try again ++ */ ++ continue; ++ if (!test_bit(STRIPE_EXPANDING, &sh2->state) || ++ test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { ++ /* must have already done this block */ ++ release_stripe(sh2); ++ continue; ++ } ++ ++ /* place all the copies on one channel */ ++ init_async_submit(&submit, 0, tx, NULL, NULL, NULL); ++ tx = async_memcpy(sh2->dev[dd_idx].page, ++ sh->dev[i].page, 0, 0, STRIPE_SIZE, ++ &submit); ++ ++ set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); ++ set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); ++ for (j = 0; j < conf->raid_disks; j++) ++ if (j != sh2->pd_idx && ++ j != sh2->qd_idx && ++ !test_bit(R5_Expanded, &sh2->dev[j].flags)) ++ break; ++ if (j == conf->raid_disks) { ++ set_bit(STRIPE_EXPAND_READY, &sh2->state); ++ set_bit(STRIPE_HANDLE, &sh2->state); ++ } ++ release_stripe(sh2); ++ ++ } ++ /* done submitting copies, wait for them to complete */ ++ async_tx_quiesce(&tx); ++} ++ ++/* ++ * handle_stripe - do things to a stripe. ++ * ++ * We lock the stripe by setting STRIPE_ACTIVE and then examine the ++ * state of various bits to see what needs to be done. ++ * Possible results: ++ * return some read requests which now have data ++ * return some write requests which are safely on storage ++ * schedule a read on some buffers ++ * schedule a write of some buffers ++ * return confirmation of parity correctness ++ * ++ */ ++ ++static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ int disks = sh->disks; ++ struct r5dev *dev; ++ int i; ++ int do_recovery = 0; ++ ++ memset(s, 0, sizeof(*s)); ++ ++ s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head; ++ s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head; ++ s->failed_num[0] = -1; ++ s->failed_num[1] = -1; ++ ++ /* Now to look around and see what can be done */ ++ rcu_read_lock(); ++ for (i=disks; i--; ) { ++ struct md_rdev *rdev; ++ sector_t first_bad; ++ int bad_sectors; ++ int is_bad = 0; ++ ++ dev = &sh->dev[i]; ++ ++ pr_debug("check %d: state 0x%lx read %p write %p written %p\n", ++ i, dev->flags, ++ dev->toread, dev->towrite, dev->written); ++ /* maybe we can reply to a read ++ * ++ * new wantfill requests are only permitted while ++ * ops_complete_biofill is guaranteed to be inactive ++ */ ++ if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && ++ !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) ++ set_bit(R5_Wantfill, &dev->flags); ++ ++ /* now count some things */ ++ if (test_bit(R5_LOCKED, &dev->flags)) ++ s->locked++; ++ if (test_bit(R5_UPTODATE, &dev->flags)) ++ s->uptodate++; ++ if (test_bit(R5_Wantcompute, &dev->flags)) { ++ s->compute++; ++ BUG_ON(s->compute > 2); ++ } ++ ++ if (test_bit(R5_Wantfill, &dev->flags)) ++ s->to_fill++; ++ else if (dev->toread) ++ s->to_read++; ++ if (dev->towrite) { ++ s->to_write++; ++ if (!test_bit(R5_OVERWRITE, &dev->flags)) ++ s->non_overwrite++; ++ } ++ if (dev->written) ++ s->written++; ++ /* Prefer to use the replacement for reads, but only ++ * if it is recovered enough and has no bad blocks. ++ */ ++ rdev = rcu_dereference(conf->disks[i].replacement); ++ if (rdev && !test_bit(Faulty, &rdev->flags) && ++ rdev->recovery_offset >= sh->sector + STRIPE_SECTORS && ++ !is_badblock(rdev, sh->sector, STRIPE_SECTORS, ++ &first_bad, &bad_sectors)) ++ set_bit(R5_ReadRepl, &dev->flags); ++ else { ++ if (rdev) ++ set_bit(R5_NeedReplace, &dev->flags); ++ rdev = rcu_dereference(conf->disks[i].rdev); ++ clear_bit(R5_ReadRepl, &dev->flags); ++ } ++ if (rdev && test_bit(Faulty, &rdev->flags)) ++ rdev = NULL; ++ if (rdev) { ++ is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, ++ &first_bad, &bad_sectors); ++ if (s->blocked_rdev == NULL ++ && (test_bit(Blocked, &rdev->flags) ++ || is_bad < 0)) { ++ if (is_bad < 0) ++ set_bit(BlockedBadBlocks, ++ &rdev->flags); ++ s->blocked_rdev = rdev; ++ atomic_inc(&rdev->nr_pending); ++ } ++ } ++ clear_bit(R5_Insync, &dev->flags); ++ if (!rdev) ++ /* Not in-sync */; ++ else if (is_bad) { ++ /* also not in-sync */ ++ if (!test_bit(WriteErrorSeen, &rdev->flags) && ++ test_bit(R5_UPTODATE, &dev->flags)) { ++ /* treat as in-sync, but with a read error ++ * which we can now try to correct ++ */ ++ set_bit(R5_Insync, &dev->flags); ++ set_bit(R5_ReadError, &dev->flags); ++ } ++ } else if (test_bit(In_sync, &rdev->flags)) ++ set_bit(R5_Insync, &dev->flags); ++ else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) ++ /* in sync if before recovery_offset */ ++ set_bit(R5_Insync, &dev->flags); ++ else if (test_bit(R5_UPTODATE, &dev->flags) && ++ test_bit(R5_Expanded, &dev->flags)) ++ /* If we've reshaped into here, we assume it is Insync. ++ * We will shortly update recovery_offset to make ++ * it official. ++ */ ++ set_bit(R5_Insync, &dev->flags); ++ ++ if (test_bit(R5_WriteError, &dev->flags)) { ++ /* This flag does not apply to '.replacement' ++ * only to .rdev, so make sure to check that*/ ++ struct md_rdev *rdev2 = rcu_dereference( ++ conf->disks[i].rdev); ++ if (rdev2 == rdev) ++ clear_bit(R5_Insync, &dev->flags); ++ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { ++ s->handle_bad_blocks = 1; ++ atomic_inc(&rdev2->nr_pending); ++ } else ++ clear_bit(R5_WriteError, &dev->flags); ++ } ++ if (test_bit(R5_MadeGood, &dev->flags)) { ++ /* This flag does not apply to '.replacement' ++ * only to .rdev, so make sure to check that*/ ++ struct md_rdev *rdev2 = rcu_dereference( ++ conf->disks[i].rdev); ++ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { ++ s->handle_bad_blocks = 1; ++ atomic_inc(&rdev2->nr_pending); ++ } else ++ clear_bit(R5_MadeGood, &dev->flags); ++ } ++ if (test_bit(R5_MadeGoodRepl, &dev->flags)) { ++ struct md_rdev *rdev2 = rcu_dereference( ++ conf->disks[i].replacement); ++ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { ++ s->handle_bad_blocks = 1; ++ atomic_inc(&rdev2->nr_pending); ++ } else ++ clear_bit(R5_MadeGoodRepl, &dev->flags); ++ } ++ if (!test_bit(R5_Insync, &dev->flags)) { ++ /* The ReadError flag will just be confusing now */ ++ clear_bit(R5_ReadError, &dev->flags); ++ clear_bit(R5_ReWrite, &dev->flags); ++ } ++ if (test_bit(R5_ReadError, &dev->flags)) ++ clear_bit(R5_Insync, &dev->flags); ++ if (!test_bit(R5_Insync, &dev->flags)) { ++ if (s->failed < 2) ++ s->failed_num[s->failed] = i; ++ s->failed++; ++ if (rdev && !test_bit(Faulty, &rdev->flags)) ++ do_recovery = 1; ++ } ++ } ++ if (test_bit(STRIPE_SYNCING, &sh->state)) { ++ /* If there is a failed device being replaced, ++ * we must be recovering. ++ * else if we are after recovery_cp, we must be syncing ++ * else if MD_RECOVERY_REQUESTED is set, we also are syncing. ++ * else we can only be replacing ++ * sync and recovery both need to read all devices, and so ++ * use the same flag. ++ */ ++ if (do_recovery || ++ sh->sector >= conf->mddev->recovery_cp || ++ test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) ++ s->syncing = 1; ++ else ++ s->replacing = 1; ++ } ++ rcu_read_unlock(); ++} ++ ++static int clear_batch_ready(struct stripe_head *sh) ++{ ++ /* Return '1' if this is a member of batch, or ++ * '0' if it is a lone stripe or a head which can now be ++ * handled. ++ */ ++ struct stripe_head *tmp; ++ if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state)) ++ return (sh->batch_head && sh->batch_head != sh); ++ spin_lock(&sh->stripe_lock); ++ if (!sh->batch_head) { ++ spin_unlock(&sh->stripe_lock); ++ return 0; ++ } ++ ++ /* ++ * this stripe could be added to a batch list before we check ++ * BATCH_READY, skips it ++ */ ++ if (sh->batch_head != sh) { ++ spin_unlock(&sh->stripe_lock); ++ return 1; ++ } ++ spin_lock(&sh->batch_lock); ++ list_for_each_entry(tmp, &sh->batch_list, batch_list) ++ clear_bit(STRIPE_BATCH_READY, &tmp->state); ++ spin_unlock(&sh->batch_lock); ++ spin_unlock(&sh->stripe_lock); ++ ++ /* ++ * BATCH_READY is cleared, no new stripes can be added. ++ * batch_list can be accessed without lock ++ */ ++ return 0; ++} ++ ++static void break_stripe_batch_list(struct stripe_head *head_sh, ++ unsigned long handle_flags) ++{ ++ struct stripe_head *sh, *next; ++ int i; ++ int do_wakeup = 0; ++ ++ list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) { ++ ++ list_del_init(&sh->batch_list); ++ ++ WARN_ON_ONCE(sh->state & ((1 << STRIPE_ACTIVE) | ++ (1 << STRIPE_SYNCING) | ++ (1 << STRIPE_REPLACED) | ++ (1 << STRIPE_PREREAD_ACTIVE) | ++ (1 << STRIPE_DELAYED) | ++ (1 << STRIPE_BIT_DELAY) | ++ (1 << STRIPE_FULL_WRITE) | ++ (1 << STRIPE_BIOFILL_RUN) | ++ (1 << STRIPE_COMPUTE_RUN) | ++ (1 << STRIPE_OPS_REQ_PENDING) | ++ (1 << STRIPE_DISCARD) | ++ (1 << STRIPE_BATCH_READY) | ++ (1 << STRIPE_BATCH_ERR) | ++ (1 << STRIPE_BITMAP_PENDING))); ++ WARN_ON_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) | ++ (1 << STRIPE_REPLACED))); ++ ++ set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS | ++ (1 << STRIPE_DEGRADED)), ++ head_sh->state & (1 << STRIPE_INSYNC)); ++ ++ sh->check_state = head_sh->check_state; ++ sh->reconstruct_state = head_sh->reconstruct_state; ++ for (i = 0; i < sh->disks; i++) { ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) ++ do_wakeup = 1; ++ sh->dev[i].flags = head_sh->dev[i].flags & ++ (~((1 << R5_WriteError) | (1 << R5_Overlap))); ++ } ++ spin_lock_irq(&sh->stripe_lock); ++ sh->batch_head = NULL; ++ spin_unlock_irq(&sh->stripe_lock); ++ if (handle_flags == 0 || ++ sh->state & handle_flags) ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++ } ++ spin_lock_irq(&head_sh->stripe_lock); ++ head_sh->batch_head = NULL; ++ spin_unlock_irq(&head_sh->stripe_lock); ++ for (i = 0; i < head_sh->disks; i++) ++ if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags)) ++ do_wakeup = 1; ++ if (head_sh->state & handle_flags) ++ set_bit(STRIPE_HANDLE, &head_sh->state); ++ ++ if (do_wakeup) ++ wake_up(&head_sh->raid_conf->wait_for_overlap); ++} ++ ++static void handle_stripe(struct stripe_head *sh) ++{ ++ struct stripe_head_state s; ++ struct r5conf *conf = sh->raid_conf; ++ int i; ++ int prexor; ++ int disks = sh->disks; ++ struct r5dev *pdev, *qdev; ++ ++ clear_bit(STRIPE_HANDLE, &sh->state); ++ if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { ++ /* already being handled, ensure it gets handled ++ * again when current action finishes */ ++ set_bit(STRIPE_HANDLE, &sh->state); ++ return; ++ } ++ ++ if (clear_batch_ready(sh) ) { ++ clear_bit_unlock(STRIPE_ACTIVE, &sh->state); ++ return; ++ } ++ ++ if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state)) ++ break_stripe_batch_list(sh, 0); ++ ++ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) { ++ spin_lock(&sh->stripe_lock); ++ /* Cannot process 'sync' concurrently with 'discard' */ ++ if (!test_bit(STRIPE_DISCARD, &sh->state) && ++ test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { ++ set_bit(STRIPE_SYNCING, &sh->state); ++ clear_bit(STRIPE_INSYNC, &sh->state); ++ clear_bit(STRIPE_REPLACED, &sh->state); ++ } ++ spin_unlock(&sh->stripe_lock); ++ } ++ clear_bit(STRIPE_DELAYED, &sh->state); ++ ++ pr_debug("handling stripe %llu, state=%#lx cnt=%d, " ++ "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", ++ (unsigned long long)sh->sector, sh->state, ++ atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, ++ sh->check_state, sh->reconstruct_state); ++ ++ analyse_stripe(sh, &s); ++ ++ if (s.handle_bad_blocks) { ++ set_bit(STRIPE_HANDLE, &sh->state); ++ goto finish; ++ } ++ ++ if (unlikely(s.blocked_rdev)) { ++ if (s.syncing || s.expanding || s.expanded || ++ s.replacing || s.to_write || s.written) { ++ set_bit(STRIPE_HANDLE, &sh->state); ++ goto finish; ++ } ++ /* There is nothing for the blocked_rdev to block */ ++ rdev_dec_pending(s.blocked_rdev, conf->mddev); ++ s.blocked_rdev = NULL; ++ } ++ ++ if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { ++ set_bit(STRIPE_OP_BIOFILL, &s.ops_request); ++ set_bit(STRIPE_BIOFILL_RUN, &sh->state); ++ } ++ ++ pr_debug("locked=%d uptodate=%d to_read=%d" ++ " to_write=%d failed=%d failed_num=%d,%d\n", ++ s.locked, s.uptodate, s.to_read, s.to_write, s.failed, ++ s.failed_num[0], s.failed_num[1]); ++ /* check if the array has lost more than max_degraded devices and, ++ * if so, some requests might need to be failed. ++ */ ++ if (s.failed > conf->max_degraded) { ++ sh->check_state = 0; ++ sh->reconstruct_state = 0; ++ break_stripe_batch_list(sh, 0); ++ if (s.to_read+s.to_write+s.written) ++ handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); ++ if (s.syncing + s.replacing) ++ handle_failed_sync(conf, sh, &s); ++ } ++ ++ /* Now we check to see if any write operations have recently ++ * completed ++ */ ++ prexor = 0; ++ if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) ++ prexor = 1; ++ if (sh->reconstruct_state == reconstruct_state_drain_result || ++ sh->reconstruct_state == reconstruct_state_prexor_drain_result) { ++ sh->reconstruct_state = reconstruct_state_idle; ++ ++ /* All the 'written' buffers and the parity block are ready to ++ * be written back to disk ++ */ ++ BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) && ++ !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)); ++ BUG_ON(sh->qd_idx >= 0 && ++ !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) && ++ !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags)); ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (test_bit(R5_LOCKED, &dev->flags) && ++ (i == sh->pd_idx || i == sh->qd_idx || ++ dev->written)) { ++ pr_debug("Writing block %d\n", i); ++ set_bit(R5_Wantwrite, &dev->flags); ++ if (prexor) ++ continue; ++ if (s.failed > 1) ++ continue; ++ if (!test_bit(R5_Insync, &dev->flags) || ++ ((i == sh->pd_idx || i == sh->qd_idx) && ++ s.failed == 0)) ++ set_bit(STRIPE_INSYNC, &sh->state); ++ } ++ } ++ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ s.dec_preread_active = 1; ++ } ++ ++ /* ++ * might be able to return some write requests if the parity blocks ++ * are safe, or on a failed drive ++ */ ++ pdev = &sh->dev[sh->pd_idx]; ++ s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) ++ || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); ++ qdev = &sh->dev[sh->qd_idx]; ++ s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) ++ || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) ++ || conf->level < 6; ++ ++ if (s.written && ++ (s.p_failed || ((test_bit(R5_Insync, &pdev->flags) ++ && !test_bit(R5_LOCKED, &pdev->flags) ++ && (test_bit(R5_UPTODATE, &pdev->flags) || ++ test_bit(R5_Discard, &pdev->flags))))) && ++ (s.q_failed || ((test_bit(R5_Insync, &qdev->flags) ++ && !test_bit(R5_LOCKED, &qdev->flags) ++ && (test_bit(R5_UPTODATE, &qdev->flags) || ++ test_bit(R5_Discard, &qdev->flags)))))) ++ handle_stripe_clean_event(conf, sh, disks, &s.return_bi); ++ ++ /* Now we might consider reading some blocks, either to check/generate ++ * parity, or to satisfy requests ++ * or to load a block that is being partially written. ++ */ ++ if (s.to_read || s.non_overwrite ++ || (conf->level == 6 && s.to_write && s.failed) ++ || (s.syncing && (s.uptodate + s.compute < disks)) ++ || s.replacing ++ || s.expanding) ++ handle_stripe_fill(sh, &s, disks); ++ ++ /* Now to consider new write requests and what else, if anything ++ * should be read. We do not handle new writes when: ++ * 1/ A 'write' operation (copy+xor) is already in flight. ++ * 2/ A 'check' operation is in flight, as it may clobber the parity ++ * block. ++ */ ++ if (s.to_write && !sh->reconstruct_state && !sh->check_state) ++ handle_stripe_dirtying(conf, sh, &s, disks); ++ ++ /* maybe we need to check and possibly fix the parity for this stripe ++ * Any reads will already have been scheduled, so we just see if enough ++ * data is available. The parity check is held off while parity ++ * dependent operations are in flight. ++ */ ++ if (sh->check_state || ++ (s.syncing && s.locked == 0 && ++ !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && ++ !test_bit(STRIPE_INSYNC, &sh->state))) { ++ if (conf->level == 6) ++ handle_parity_checks6(conf, sh, &s, disks); ++ else ++ handle_parity_checks5(conf, sh, &s, disks); ++ } ++ ++ if ((s.replacing || s.syncing) && s.locked == 0 ++ && !test_bit(STRIPE_COMPUTE_RUN, &sh->state) ++ && !test_bit(STRIPE_REPLACED, &sh->state)) { ++ /* Write out to replacement devices where possible */ ++ for (i = 0; i < conf->raid_disks; i++) ++ if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) { ++ WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags)); ++ set_bit(R5_WantReplace, &sh->dev[i].flags); ++ set_bit(R5_LOCKED, &sh->dev[i].flags); ++ s.locked++; ++ } ++ if (s.replacing) ++ set_bit(STRIPE_INSYNC, &sh->state); ++ set_bit(STRIPE_REPLACED, &sh->state); ++ } ++ if ((s.syncing || s.replacing) && s.locked == 0 && ++ !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && ++ test_bit(STRIPE_INSYNC, &sh->state)) { ++ md_done_sync(conf->mddev, STRIPE_SECTORS, 1); ++ clear_bit(STRIPE_SYNCING, &sh->state); ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) ++ wake_up(&conf->wait_for_overlap); ++ } ++ ++ /* If the failed drives are just a ReadError, then we might need ++ * to progress the repair/check process ++ */ ++ if (s.failed <= conf->max_degraded && !conf->mddev->ro) ++ for (i = 0; i < s.failed; i++) { ++ struct r5dev *dev = &sh->dev[s.failed_num[i]]; ++ if (test_bit(R5_ReadError, &dev->flags) ++ && !test_bit(R5_LOCKED, &dev->flags) ++ && test_bit(R5_UPTODATE, &dev->flags) ++ ) { ++ if (!test_bit(R5_ReWrite, &dev->flags)) { ++ set_bit(R5_Wantwrite, &dev->flags); ++ set_bit(R5_ReWrite, &dev->flags); ++ set_bit(R5_LOCKED, &dev->flags); ++ s.locked++; ++ } else { ++ /* let's read it back */ ++ set_bit(R5_Wantread, &dev->flags); ++ set_bit(R5_LOCKED, &dev->flags); ++ s.locked++; ++ } ++ } ++ } ++ ++ /* Finish reconstruct operations initiated by the expansion process */ ++ if (sh->reconstruct_state == reconstruct_state_result) { ++ struct stripe_head *sh_src ++ = get_active_stripe(conf, sh->sector, 1, 1, 1); ++ if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { ++ /* sh cannot be written until sh_src has been read. ++ * so arrange for sh to be delayed a little ++ */ ++ set_bit(STRIPE_DELAYED, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, ++ &sh_src->state)) ++ atomic_inc(&conf->preread_active_stripes); ++ release_stripe(sh_src); ++ goto finish; ++ } ++ if (sh_src) ++ release_stripe(sh_src); ++ ++ sh->reconstruct_state = reconstruct_state_idle; ++ clear_bit(STRIPE_EXPANDING, &sh->state); ++ for (i = conf->raid_disks; i--; ) { ++ set_bit(R5_Wantwrite, &sh->dev[i].flags); ++ set_bit(R5_LOCKED, &sh->dev[i].flags); ++ s.locked++; ++ } ++ } ++ ++ if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && ++ !sh->reconstruct_state) { ++ /* Need to write out all blocks after computing parity */ ++ sh->disks = conf->raid_disks; ++ stripe_set_idx(sh->sector, conf, 0, sh); ++ schedule_reconstruction(sh, &s, 1, 1); ++ } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { ++ clear_bit(STRIPE_EXPAND_READY, &sh->state); ++ atomic_dec(&conf->reshape_stripes); ++ wake_up(&conf->wait_for_overlap); ++ md_done_sync(conf->mddev, STRIPE_SECTORS, 1); ++ } ++ ++ if (s.expanding && s.locked == 0 && ++ !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) ++ handle_stripe_expansion(conf, sh); ++ ++finish: ++ /* wait for this device to become unblocked */ ++ if (unlikely(s.blocked_rdev)) { ++ if (conf->mddev->external) ++ md_wait_for_blocked_rdev(s.blocked_rdev, ++ conf->mddev); ++ else ++ /* Internal metadata will immediately ++ * be written by raid5d, so we don't ++ * need to wait here. ++ */ ++ rdev_dec_pending(s.blocked_rdev, ++ conf->mddev); ++ } ++ ++ if (s.handle_bad_blocks) ++ for (i = disks; i--; ) { ++ struct md_rdev *rdev; ++ struct r5dev *dev = &sh->dev[i]; ++ if (test_and_clear_bit(R5_WriteError, &dev->flags)) { ++ /* We own a safe reference to the rdev */ ++ rdev = conf->disks[i].rdev; ++ if (!rdev_set_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0)) ++ md_error(conf->mddev, rdev); ++ rdev_dec_pending(rdev, conf->mddev); ++ } ++ if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { ++ rdev = conf->disks[i].rdev; ++ rdev_clear_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0); ++ rdev_dec_pending(rdev, conf->mddev); ++ } ++ if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { ++ rdev = conf->disks[i].replacement; ++ if (!rdev) ++ /* rdev have been moved down */ ++ rdev = conf->disks[i].rdev; ++ rdev_clear_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0); ++ rdev_dec_pending(rdev, conf->mddev); ++ } ++ } ++ ++ if (s.ops_request) ++ raid_run_ops(sh, s.ops_request); ++ ++ ops_run_io(sh, &s); ++ ++ if (s.dec_preread_active) { ++ /* We delay this until after ops_run_io so that if make_request ++ * is waiting on a flush, it won't continue until the writes ++ * have actually been submitted. ++ */ ++ atomic_dec(&conf->preread_active_stripes); ++ if (atomic_read(&conf->preread_active_stripes) < ++ IO_THRESHOLD) ++ md_wakeup_thread(conf->mddev->thread); ++ } ++ ++ return_io(s.return_bi); ++ ++ clear_bit_unlock(STRIPE_ACTIVE, &sh->state); ++} ++ ++static void raid5_activate_delayed(struct r5conf *conf) ++{ ++ if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { ++ while (!list_empty(&conf->delayed_list)) { ++ struct list_head *l = conf->delayed_list.next; ++ struct stripe_head *sh; ++ sh = list_entry(l, struct stripe_head, lru); ++ list_del_init(l); ++ clear_bit(STRIPE_DELAYED, &sh->state); ++ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ atomic_inc(&conf->preread_active_stripes); ++ list_add_tail(&sh->lru, &conf->hold_list); ++ raid5_wakeup_stripe_thread(sh); ++ } ++ } ++} ++ ++static void activate_bit_delay(struct r5conf *conf, ++ struct list_head *temp_inactive_list) ++{ ++ /* device_lock is held */ ++ struct list_head head; ++ list_add(&head, &conf->bitmap_list); ++ list_del_init(&conf->bitmap_list); ++ while (!list_empty(&head)) { ++ struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); ++ int hash; ++ list_del_init(&sh->lru); ++ atomic_inc(&sh->count); ++ hash = sh->hash_lock_index; ++ __release_stripe(conf, sh, &temp_inactive_list[hash]); ++ } ++} ++ ++static int raid5_congested(struct mddev *mddev, int bits) ++{ ++ struct r5conf *conf = mddev->private; ++ ++ /* No difference between reads and writes. Just check ++ * how busy the stripe_cache is ++ */ ++ ++ if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) ++ return 1; ++ if (conf->quiesce) ++ return 1; ++ if (atomic_read(&conf->empty_inactive_list_nr)) ++ return 1; ++ ++ return 0; ++} ++ ++/* We want read requests to align with chunks where possible, ++ * but write requests don't need to. ++ */ ++static int raid5_mergeable_bvec(struct mddev *mddev, ++ struct bvec_merge_data *bvm, ++ struct bio_vec *biovec) ++{ ++ sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); ++ int max; ++ unsigned int chunk_sectors = mddev->chunk_sectors; ++ unsigned int bio_sectors = bvm->bi_size >> 9; ++ ++ /* ++ * always allow writes to be mergeable, read as well if array ++ * is degraded as we'll go through stripe cache anyway. ++ */ ++ if ((bvm->bi_rw & 1) == WRITE || mddev->degraded) ++ return biovec->bv_len; ++ ++ if (mddev->new_chunk_sectors < mddev->chunk_sectors) ++ chunk_sectors = mddev->new_chunk_sectors; ++ max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; ++ if (max < 0) max = 0; ++ if (max <= biovec->bv_len && bio_sectors == 0) ++ return biovec->bv_len; ++ else ++ return max; ++} ++ ++static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) ++{ ++ sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); ++ unsigned int chunk_sectors = mddev->chunk_sectors; ++ unsigned int bio_sectors = bio_sectors(bio); ++ ++ if (mddev->new_chunk_sectors < mddev->chunk_sectors) ++ chunk_sectors = mddev->new_chunk_sectors; ++ return chunk_sectors >= ++ ((sector & (chunk_sectors - 1)) + bio_sectors); ++} ++ ++/* ++ * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) ++ * later sampled by raid5d. ++ */ ++static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&conf->device_lock, flags); ++ ++ bi->bi_next = conf->retry_read_aligned_list; ++ conf->retry_read_aligned_list = bi; ++ ++ spin_unlock_irqrestore(&conf->device_lock, flags); ++ md_wakeup_thread(conf->mddev->thread); ++} ++ ++static struct bio *remove_bio_from_retry(struct r5conf *conf) ++{ ++ struct bio *bi; ++ ++ bi = conf->retry_read_aligned; ++ if (bi) { ++ conf->retry_read_aligned = NULL; ++ return bi; ++ } ++ bi = conf->retry_read_aligned_list; ++ if(bi) { ++ conf->retry_read_aligned_list = bi->bi_next; ++ bi->bi_next = NULL; ++ /* ++ * this sets the active strip count to 1 and the processed ++ * strip count to zero (upper 8 bits) ++ */ ++ raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */ ++ } ++ ++ return bi; ++} ++ ++/* ++ * The "raid5_align_endio" should check if the read succeeded and if it ++ * did, call bio_endio on the original bio (having bio_put the new bio ++ * first). ++ * If the read failed.. ++ */ ++static void raid5_align_endio(struct bio *bi, int error) ++{ ++ struct bio* raid_bi = bi->bi_private; ++ struct mddev *mddev; ++ struct r5conf *conf; ++ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); ++ struct md_rdev *rdev; ++ ++ bio_put(bi); ++ ++ rdev = (void*)raid_bi->bi_next; ++ raid_bi->bi_next = NULL; ++ mddev = rdev->mddev; ++ conf = mddev->private; ++ ++ rdev_dec_pending(rdev, conf->mddev); ++ ++ if (!error && uptodate) { ++ trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), ++ raid_bi, 0); ++ bio_endio(raid_bi, 0); ++ if (atomic_dec_and_test(&conf->active_aligned_reads)) ++ wake_up(&conf->wait_for_stripe); ++ return; ++ } ++ ++ pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); ++ ++ add_bio_to_retry(raid_bi, conf); ++} ++ ++static int bio_fits_rdev(struct bio *bi) ++{ ++ struct request_queue *q = bdev_get_queue(bi->bi_bdev); ++ ++ if (bio_sectors(bi) > queue_max_sectors(q)) ++ return 0; ++ blk_recount_segments(q, bi); ++ if (bi->bi_phys_segments > queue_max_segments(q)) ++ return 0; ++ ++ if (q->merge_bvec_fn) ++ /* it's too hard to apply the merge_bvec_fn at this stage, ++ * just just give up ++ */ ++ return 0; ++ ++ return 1; ++} ++ ++static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) ++{ ++ struct r5conf *conf = mddev->private; ++ int dd_idx; ++ struct bio* align_bi; ++ struct md_rdev *rdev; ++ sector_t end_sector; ++ ++ if (!in_chunk_boundary(mddev, raid_bio)) { ++ pr_debug("chunk_aligned_read : non aligned\n"); ++ return 0; ++ } ++ /* ++ * use bio_clone_mddev to make a copy of the bio ++ */ ++ align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); ++ if (!align_bi) ++ return 0; ++ /* ++ * set bi_end_io to a new function, and set bi_private to the ++ * original bio. ++ */ ++ align_bi->bi_end_io = raid5_align_endio; ++ align_bi->bi_private = raid_bio; ++ /* ++ * compute position ++ */ ++ align_bi->bi_iter.bi_sector = ++ raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, ++ 0, &dd_idx, NULL); ++ ++ end_sector = bio_end_sector(align_bi); ++ rcu_read_lock(); ++ rdev = rcu_dereference(conf->disks[dd_idx].replacement); ++ if (!rdev || test_bit(Faulty, &rdev->flags) || ++ rdev->recovery_offset < end_sector) { ++ rdev = rcu_dereference(conf->disks[dd_idx].rdev); ++ if (rdev && ++ (test_bit(Faulty, &rdev->flags) || ++ !(test_bit(In_sync, &rdev->flags) || ++ rdev->recovery_offset >= end_sector))) ++ rdev = NULL; ++ } ++ if (rdev) { ++ sector_t first_bad; ++ int bad_sectors; ++ ++ atomic_inc(&rdev->nr_pending); ++ rcu_read_unlock(); ++ raid_bio->bi_next = (void*)rdev; ++ align_bi->bi_bdev = rdev->bdev; ++ __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags); ++ ++ if (!bio_fits_rdev(align_bi) || ++ is_badblock(rdev, align_bi->bi_iter.bi_sector, ++ bio_sectors(align_bi), ++ &first_bad, &bad_sectors)) { ++ /* too big in some way, or has a known bad block */ ++ bio_put(align_bi); ++ rdev_dec_pending(rdev, mddev); ++ return 0; ++ } ++ ++ /* No reshape active, so we can trust rdev->data_offset */ ++ align_bi->bi_iter.bi_sector += rdev->data_offset; ++ ++ spin_lock_irq(&conf->device_lock); ++ wait_event_lock_irq(conf->wait_for_stripe, ++ conf->quiesce == 0, ++ conf->device_lock); ++ atomic_inc(&conf->active_aligned_reads); ++ spin_unlock_irq(&conf->device_lock); ++ ++ if (mddev->gendisk) ++ trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev), ++ align_bi, disk_devt(mddev->gendisk), ++ raid_bio->bi_iter.bi_sector); ++ generic_make_request(align_bi); ++ return 1; ++ } else { ++ rcu_read_unlock(); ++ bio_put(align_bi); ++ return 0; ++ } ++} ++ ++/* __get_priority_stripe - get the next stripe to process ++ * ++ * Full stripe writes are allowed to pass preread active stripes up until ++ * the bypass_threshold is exceeded. In general the bypass_count ++ * increments when the handle_list is handled before the hold_list; however, it ++ * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a ++ * stripe with in flight i/o. The bypass_count will be reset when the ++ * head of the hold_list has changed, i.e. the head was promoted to the ++ * handle_list. ++ */ ++static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group) ++{ ++ struct stripe_head *sh = NULL, *tmp; ++ struct list_head *handle_list = NULL; ++ struct r5worker_group *wg = NULL; ++ ++ if (conf->worker_cnt_per_group == 0) { ++ handle_list = &conf->handle_list; ++ } else if (group != ANY_GROUP) { ++ handle_list = &conf->worker_groups[group].handle_list; ++ wg = &conf->worker_groups[group]; ++ } else { ++ int i; ++ for (i = 0; i < conf->group_cnt; i++) { ++ handle_list = &conf->worker_groups[i].handle_list; ++ wg = &conf->worker_groups[i]; ++ if (!list_empty(handle_list)) ++ break; ++ } ++ } ++ ++ pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", ++ __func__, ++ list_empty(handle_list) ? "empty" : "busy", ++ list_empty(&conf->hold_list) ? "empty" : "busy", ++ atomic_read(&conf->pending_full_writes), conf->bypass_count); ++ ++ if (!list_empty(handle_list)) { ++ sh = list_entry(handle_list->next, typeof(*sh), lru); ++ ++ if (list_empty(&conf->hold_list)) ++ conf->bypass_count = 0; ++ else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { ++ if (conf->hold_list.next == conf->last_hold) ++ conf->bypass_count++; ++ else { ++ conf->last_hold = conf->hold_list.next; ++ conf->bypass_count -= conf->bypass_threshold; ++ if (conf->bypass_count < 0) ++ conf->bypass_count = 0; ++ } ++ } ++ } else if (!list_empty(&conf->hold_list) && ++ ((conf->bypass_threshold && ++ conf->bypass_count > conf->bypass_threshold) || ++ atomic_read(&conf->pending_full_writes) == 0)) { ++ ++ list_for_each_entry(tmp, &conf->hold_list, lru) { ++ if (conf->worker_cnt_per_group == 0 || ++ group == ANY_GROUP || ++ !cpu_online(tmp->cpu) || ++ cpu_to_group(tmp->cpu) == group) { ++ sh = tmp; ++ break; ++ } ++ } ++ ++ if (sh) { ++ conf->bypass_count -= conf->bypass_threshold; ++ if (conf->bypass_count < 0) ++ conf->bypass_count = 0; ++ } ++ wg = NULL; ++ } ++ ++ if (!sh) ++ return NULL; ++ ++ if (wg) { ++ wg->stripes_cnt--; ++ sh->group = NULL; ++ } ++ list_del_init(&sh->lru); ++ BUG_ON(atomic_inc_return(&sh->count) != 1); ++ return sh; ++} ++ ++struct raid5_plug_cb { ++ struct blk_plug_cb cb; ++ struct list_head list; ++ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; ++}; ++ ++static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule) ++{ ++ struct raid5_plug_cb *cb = container_of( ++ blk_cb, struct raid5_plug_cb, cb); ++ struct stripe_head *sh; ++ struct mddev *mddev = cb->cb.data; ++ struct r5conf *conf = mddev->private; ++ int cnt = 0; ++ int hash; ++ ++ if (cb->list.next && !list_empty(&cb->list)) { ++ spin_lock_irq(&conf->device_lock); ++ while (!list_empty(&cb->list)) { ++ sh = list_first_entry(&cb->list, struct stripe_head, lru); ++ list_del_init(&sh->lru); ++ /* ++ * avoid race release_stripe_plug() sees ++ * STRIPE_ON_UNPLUG_LIST clear but the stripe ++ * is still in our list ++ */ ++ smp_mb__before_atomic(); ++ clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); ++ /* ++ * STRIPE_ON_RELEASE_LIST could be set here. In that ++ * case, the count is always > 1 here ++ */ ++ hash = sh->hash_lock_index; ++ __release_stripe(conf, sh, &cb->temp_inactive_list[hash]); ++ cnt++; ++ } ++ spin_unlock_irq(&conf->device_lock); ++ } ++ release_inactive_stripe_list(conf, cb->temp_inactive_list, ++ NR_STRIPE_HASH_LOCKS); ++ if (mddev->queue) ++ trace_block_unplug(mddev->queue, cnt, !from_schedule); ++ kfree(cb); ++} ++ ++static void release_stripe_plug(struct mddev *mddev, ++ struct stripe_head *sh) ++{ ++ struct blk_plug_cb *blk_cb = blk_check_plugged( ++ raid5_unplug, mddev, ++ sizeof(struct raid5_plug_cb)); ++ struct raid5_plug_cb *cb; ++ ++ if (!blk_cb) { ++ release_stripe(sh); ++ return; ++ } ++ ++ cb = container_of(blk_cb, struct raid5_plug_cb, cb); ++ ++ if (cb->list.next == NULL) { ++ int i; ++ INIT_LIST_HEAD(&cb->list); ++ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) ++ INIT_LIST_HEAD(cb->temp_inactive_list + i); ++ } ++ ++ if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)) ++ list_add_tail(&sh->lru, &cb->list); ++ else ++ release_stripe(sh); ++} ++ ++static void make_discard_request(struct mddev *mddev, struct bio *bi) ++{ ++ struct r5conf *conf = mddev->private; ++ sector_t logical_sector, last_sector; ++ struct stripe_head *sh; ++ int remaining; ++ int stripe_sectors; ++ ++ if (mddev->reshape_position != MaxSector) ++ /* Skip discard while reshape is happening */ ++ return; ++ ++ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); ++ last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9); ++ ++ bi->bi_next = NULL; ++ bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ ++ ++ stripe_sectors = conf->chunk_sectors * ++ (conf->raid_disks - conf->max_degraded); ++ logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, ++ stripe_sectors); ++ sector_div(last_sector, stripe_sectors); ++ ++ logical_sector *= conf->chunk_sectors; ++ last_sector *= conf->chunk_sectors; ++ ++ for (; logical_sector < last_sector; ++ logical_sector += STRIPE_SECTORS) { ++ DEFINE_WAIT(w); ++ int d; ++ again: ++ sh = get_active_stripe(conf, logical_sector, 0, 0, 0); ++ prepare_to_wait(&conf->wait_for_overlap, &w, ++ TASK_UNINTERRUPTIBLE); ++ set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); ++ if (test_bit(STRIPE_SYNCING, &sh->state)) { ++ release_stripe(sh); ++ schedule(); ++ goto again; ++ } ++ clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); ++ spin_lock_irq(&sh->stripe_lock); ++ for (d = 0; d < conf->raid_disks; d++) { ++ if (d == sh->pd_idx || d == sh->qd_idx) ++ continue; ++ if (sh->dev[d].towrite || sh->dev[d].toread) { ++ set_bit(R5_Overlap, &sh->dev[d].flags); ++ spin_unlock_irq(&sh->stripe_lock); ++ release_stripe(sh); ++ schedule(); ++ goto again; ++ } ++ } ++ set_bit(STRIPE_DISCARD, &sh->state); ++ finish_wait(&conf->wait_for_overlap, &w); ++ sh->overwrite_disks = 0; ++ for (d = 0; d < conf->raid_disks; d++) { ++ if (d == sh->pd_idx || d == sh->qd_idx) ++ continue; ++ sh->dev[d].towrite = bi; ++ set_bit(R5_OVERWRITE, &sh->dev[d].flags); ++ raid5_inc_bi_active_stripes(bi); ++ sh->overwrite_disks++; ++ } ++ spin_unlock_irq(&sh->stripe_lock); ++ if (conf->mddev->bitmap) { ++ for (d = 0; ++ d < conf->raid_disks - conf->max_degraded; ++ d++) ++ bitmap_startwrite(mddev->bitmap, ++ sh->sector, ++ STRIPE_SECTORS, ++ 0); ++ sh->bm_seq = conf->seq_flush + 1; ++ set_bit(STRIPE_BIT_DELAY, &sh->state); ++ } ++ ++ set_bit(STRIPE_HANDLE, &sh->state); ++ clear_bit(STRIPE_DELAYED, &sh->state); ++ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ atomic_inc(&conf->preread_active_stripes); ++ release_stripe_plug(mddev, sh); ++ } ++ ++ remaining = raid5_dec_bi_active_stripes(bi); ++ if (remaining == 0) { ++ md_write_end(mddev); ++ bio_endio(bi, 0); ++ } ++} ++ ++static void make_request(struct mddev *mddev, struct bio * bi) ++{ ++ struct r5conf *conf = mddev->private; ++ int dd_idx; ++ sector_t new_sector; ++ sector_t logical_sector, last_sector; ++ struct stripe_head *sh; ++ const int rw = bio_data_dir(bi); ++ int remaining; ++ DEFINE_WAIT(w); ++ bool do_prepare; ++ ++ if (unlikely(bi->bi_rw & REQ_FLUSH)) { ++ md_flush_request(mddev, bi); ++ return; ++ } ++ ++ md_write_start(mddev, bi); ++ ++ /* ++ * If array is degraded, better not do chunk aligned read because ++ * later we might have to read it again in order to reconstruct ++ * data on failed drives. ++ */ ++ if (rw == READ && mddev->degraded == 0 && ++ mddev->reshape_position == MaxSector && ++ chunk_aligned_read(mddev,bi)) ++ return; ++ ++ if (unlikely(bi->bi_rw & REQ_DISCARD)) { ++ make_discard_request(mddev, bi); ++ return; ++ } ++ ++ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); ++ last_sector = bio_end_sector(bi); ++ bi->bi_next = NULL; ++ bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ ++ ++ prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); ++ for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { ++ int previous; ++ int seq; ++ ++ do_prepare = false; ++ retry: ++ seq = read_seqcount_begin(&conf->gen_lock); ++ previous = 0; ++ if (do_prepare) ++ prepare_to_wait(&conf->wait_for_overlap, &w, ++ TASK_UNINTERRUPTIBLE); ++ if (unlikely(conf->reshape_progress != MaxSector)) { ++ /* spinlock is needed as reshape_progress may be ++ * 64bit on a 32bit platform, and so it might be ++ * possible to see a half-updated value ++ * Of course reshape_progress could change after ++ * the lock is dropped, so once we get a reference ++ * to the stripe that we think it is, we will have ++ * to check again. ++ */ ++ spin_lock_irq(&conf->device_lock); ++ if (mddev->reshape_backwards ++ ? logical_sector < conf->reshape_progress ++ : logical_sector >= conf->reshape_progress) { ++ previous = 1; ++ } else { ++ if (mddev->reshape_backwards ++ ? logical_sector < conf->reshape_safe ++ : logical_sector >= conf->reshape_safe) { ++ spin_unlock_irq(&conf->device_lock); ++ schedule(); ++ do_prepare = true; ++ goto retry; ++ } ++ } ++ spin_unlock_irq(&conf->device_lock); ++ } ++ ++ new_sector = raid5_compute_sector(conf, logical_sector, ++ previous, ++ &dd_idx, NULL); ++ pr_debug("raid456: make_request, sector %llu logical %llu\n", ++ (unsigned long long)new_sector, ++ (unsigned long long)logical_sector); ++ ++ sh = get_active_stripe(conf, new_sector, previous, ++ (bi->bi_rw&RWA_MASK), 0); ++ if (sh) { ++ if (unlikely(previous)) { ++ /* expansion might have moved on while waiting for a ++ * stripe, so we must do the range check again. ++ * Expansion could still move past after this ++ * test, but as we are holding a reference to ++ * 'sh', we know that if that happens, ++ * STRIPE_EXPANDING will get set and the expansion ++ * won't proceed until we finish with the stripe. ++ */ ++ int must_retry = 0; ++ spin_lock_irq(&conf->device_lock); ++ if (mddev->reshape_backwards ++ ? logical_sector >= conf->reshape_progress ++ : logical_sector < conf->reshape_progress) ++ /* mismatch, need to try again */ ++ must_retry = 1; ++ spin_unlock_irq(&conf->device_lock); ++ if (must_retry) { ++ release_stripe(sh); ++ schedule(); ++ do_prepare = true; ++ goto retry; ++ } ++ } ++ if (read_seqcount_retry(&conf->gen_lock, seq)) { ++ /* Might have got the wrong stripe_head ++ * by accident ++ */ ++ release_stripe(sh); ++ goto retry; ++ } ++ ++ if (rw == WRITE && ++ logical_sector >= mddev->suspend_lo && ++ logical_sector < mddev->suspend_hi) { ++ release_stripe(sh); ++ /* As the suspend_* range is controlled by ++ * userspace, we want an interruptible ++ * wait. ++ */ ++ flush_signals(current); ++ prepare_to_wait(&conf->wait_for_overlap, ++ &w, TASK_INTERRUPTIBLE); ++ if (logical_sector >= mddev->suspend_lo && ++ logical_sector < mddev->suspend_hi) { ++ schedule(); ++ do_prepare = true; ++ } ++ goto retry; ++ } ++ ++ if (test_bit(STRIPE_EXPANDING, &sh->state) || ++ !add_stripe_bio(sh, bi, dd_idx, rw, previous)) { ++ /* Stripe is busy expanding or ++ * add failed due to overlap. Flush everything ++ * and wait a while ++ */ ++ md_wakeup_thread(mddev->thread); ++ release_stripe(sh); ++ schedule(); ++ do_prepare = true; ++ goto retry; ++ } ++ set_bit(STRIPE_HANDLE, &sh->state); ++ clear_bit(STRIPE_DELAYED, &sh->state); ++ if ((!sh->batch_head || sh == sh->batch_head) && ++ (bi->bi_rw & REQ_SYNC) && ++ !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ atomic_inc(&conf->preread_active_stripes); ++ release_stripe_plug(mddev, sh); ++ } else { ++ /* cannot get stripe for read-ahead, just give-up */ ++ clear_bit(BIO_UPTODATE, &bi->bi_flags); ++ break; ++ } ++ } ++ finish_wait(&conf->wait_for_overlap, &w); ++ ++ remaining = raid5_dec_bi_active_stripes(bi); ++ if (remaining == 0) { ++ ++ if ( rw == WRITE ) ++ md_write_end(mddev); ++ ++ trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), ++ bi, 0); ++ bio_endio(bi, 0); ++ } ++} ++ ++static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks); ++ ++static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped) ++{ ++ /* reshaping is quite different to recovery/resync so it is ++ * handled quite separately ... here. ++ * ++ * On each call to sync_request, we gather one chunk worth of ++ * destination stripes and flag them as expanding. ++ * Then we find all the source stripes and request reads. ++ * As the reads complete, handle_stripe will copy the data ++ * into the destination stripe and release that stripe. ++ */ ++ struct r5conf *conf = mddev->private; ++ struct stripe_head *sh; ++ sector_t first_sector, last_sector; ++ int raid_disks = conf->previous_raid_disks; ++ int data_disks = raid_disks - conf->max_degraded; ++ int new_data_disks = conf->raid_disks - conf->max_degraded; ++ int i; ++ int dd_idx; ++ sector_t writepos, readpos, safepos; ++ sector_t stripe_addr; ++ int reshape_sectors; ++ struct list_head stripes; ++ ++ if (sector_nr == 0) { ++ /* If restarting in the middle, skip the initial sectors */ ++ if (mddev->reshape_backwards && ++ conf->reshape_progress < raid5_size(mddev, 0, 0)) { ++ sector_nr = raid5_size(mddev, 0, 0) ++ - conf->reshape_progress; ++ } else if (!mddev->reshape_backwards && ++ conf->reshape_progress > 0) ++ sector_nr = conf->reshape_progress; ++ sector_div(sector_nr, new_data_disks); ++ if (sector_nr) { ++ mddev->curr_resync_completed = sector_nr; ++ sysfs_notify(&mddev->kobj, NULL, "sync_completed"); ++ *skipped = 1; ++ return sector_nr; ++ } ++ } ++ ++ /* We need to process a full chunk at a time. ++ * If old and new chunk sizes differ, we need to process the ++ * largest of these ++ */ ++ if (mddev->new_chunk_sectors > mddev->chunk_sectors) ++ reshape_sectors = mddev->new_chunk_sectors; ++ else ++ reshape_sectors = mddev->chunk_sectors; ++ ++ /* We update the metadata at least every 10 seconds, or when ++ * the data about to be copied would over-write the source of ++ * the data at the front of the range. i.e. one new_stripe ++ * along from reshape_progress new_maps to after where ++ * reshape_safe old_maps to ++ */ ++ writepos = conf->reshape_progress; ++ sector_div(writepos, new_data_disks); ++ readpos = conf->reshape_progress; ++ sector_div(readpos, data_disks); ++ safepos = conf->reshape_safe; ++ sector_div(safepos, data_disks); ++ if (mddev->reshape_backwards) { ++ writepos -= min_t(sector_t, reshape_sectors, writepos); ++ readpos += reshape_sectors; ++ safepos += reshape_sectors; ++ } else { ++ writepos += reshape_sectors; ++ readpos -= min_t(sector_t, reshape_sectors, readpos); ++ safepos -= min_t(sector_t, reshape_sectors, safepos); ++ } ++ ++ /* Having calculated the 'writepos' possibly use it ++ * to set 'stripe_addr' which is where we will write to. ++ */ ++ if (mddev->reshape_backwards) { ++ BUG_ON(conf->reshape_progress == 0); ++ stripe_addr = writepos; ++ BUG_ON((mddev->dev_sectors & ++ ~((sector_t)reshape_sectors - 1)) ++ - reshape_sectors - stripe_addr ++ != sector_nr); ++ } else { ++ BUG_ON(writepos != sector_nr + reshape_sectors); ++ stripe_addr = sector_nr; ++ } ++ ++ /* 'writepos' is the most advanced device address we might write. ++ * 'readpos' is the least advanced device address we might read. ++ * 'safepos' is the least address recorded in the metadata as having ++ * been reshaped. ++ * If there is a min_offset_diff, these are adjusted either by ++ * increasing the safepos/readpos if diff is negative, or ++ * increasing writepos if diff is positive. ++ * If 'readpos' is then behind 'writepos', there is no way that we can ++ * ensure safety in the face of a crash - that must be done by userspace ++ * making a backup of the data. So in that case there is no particular ++ * rush to update metadata. ++ * Otherwise if 'safepos' is behind 'writepos', then we really need to ++ * update the metadata to advance 'safepos' to match 'readpos' so that ++ * we can be safe in the event of a crash. ++ * So we insist on updating metadata if safepos is behind writepos and ++ * readpos is beyond writepos. ++ * In any case, update the metadata every 10 seconds. ++ * Maybe that number should be configurable, but I'm not sure it is ++ * worth it.... maybe it could be a multiple of safemode_delay??? ++ */ ++ if (conf->min_offset_diff < 0) { ++ safepos += -conf->min_offset_diff; ++ readpos += -conf->min_offset_diff; ++ } else ++ writepos += conf->min_offset_diff; ++ ++ if ((mddev->reshape_backwards ++ ? (safepos > writepos && readpos < writepos) ++ : (safepos < writepos && readpos > writepos)) || ++ time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { ++ /* Cannot proceed until we've updated the superblock... */ ++ wait_event(conf->wait_for_overlap, ++ atomic_read(&conf->reshape_stripes)==0 ++ || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); ++ if (atomic_read(&conf->reshape_stripes) != 0) ++ return 0; ++ mddev->reshape_position = conf->reshape_progress; ++ mddev->curr_resync_completed = sector_nr; ++ conf->reshape_checkpoint = jiffies; ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ md_wakeup_thread(mddev->thread); ++ wait_event(mddev->sb_wait, mddev->flags == 0 || ++ test_bit(MD_RECOVERY_INTR, &mddev->recovery)); ++ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) ++ return 0; ++ spin_lock_irq(&conf->device_lock); ++ conf->reshape_safe = mddev->reshape_position; ++ spin_unlock_irq(&conf->device_lock); ++ wake_up(&conf->wait_for_overlap); ++ sysfs_notify(&mddev->kobj, NULL, "sync_completed"); ++ } ++ ++ INIT_LIST_HEAD(&stripes); ++ for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { ++ int j; ++ int skipped_disk = 0; ++ sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); ++ set_bit(STRIPE_EXPANDING, &sh->state); ++ atomic_inc(&conf->reshape_stripes); ++ /* If any of this stripe is beyond the end of the old ++ * array, then we need to zero those blocks ++ */ ++ for (j=sh->disks; j--;) { ++ sector_t s; ++ if (j == sh->pd_idx) ++ continue; ++ if (conf->level == 6 && ++ j == sh->qd_idx) ++ continue; ++ s = compute_blocknr(sh, j, 0); ++ if (s < raid5_size(mddev, 0, 0)) { ++ skipped_disk = 1; ++ continue; ++ } ++ memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); ++ set_bit(R5_Expanded, &sh->dev[j].flags); ++ set_bit(R5_UPTODATE, &sh->dev[j].flags); ++ } ++ if (!skipped_disk) { ++ set_bit(STRIPE_EXPAND_READY, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ } ++ list_add(&sh->lru, &stripes); ++ } ++ spin_lock_irq(&conf->device_lock); ++ if (mddev->reshape_backwards) ++ conf->reshape_progress -= reshape_sectors * new_data_disks; ++ else ++ conf->reshape_progress += reshape_sectors * new_data_disks; ++ spin_unlock_irq(&conf->device_lock); ++ /* Ok, those stripe are ready. We can start scheduling ++ * reads on the source stripes. ++ * The source stripes are determined by mapping the first and last ++ * block on the destination stripes. ++ */ ++ first_sector = ++ raid5_compute_sector(conf, stripe_addr*(new_data_disks), ++ 1, &dd_idx, NULL); ++ last_sector = ++ raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) ++ * new_data_disks - 1), ++ 1, &dd_idx, NULL); ++ if (last_sector >= mddev->dev_sectors) ++ last_sector = mddev->dev_sectors - 1; ++ while (first_sector <= last_sector) { ++ sh = get_active_stripe(conf, first_sector, 1, 0, 1); ++ set_bit(STRIPE_EXPAND_SOURCE, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++ first_sector += STRIPE_SECTORS; ++ } ++ /* Now that the sources are clearly marked, we can release ++ * the destination stripes ++ */ ++ while (!list_empty(&stripes)) { ++ sh = list_entry(stripes.next, struct stripe_head, lru); ++ list_del_init(&sh->lru); ++ release_stripe(sh); ++ } ++ /* If this takes us to the resync_max point where we have to pause, ++ * then we need to write out the superblock. ++ */ ++ sector_nr += reshape_sectors; ++ if ((sector_nr - mddev->curr_resync_completed) * 2 ++ >= mddev->resync_max - mddev->curr_resync_completed) { ++ /* Cannot proceed until we've updated the superblock... */ ++ wait_event(conf->wait_for_overlap, ++ atomic_read(&conf->reshape_stripes) == 0 ++ || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); ++ if (atomic_read(&conf->reshape_stripes) != 0) ++ goto ret; ++ mddev->reshape_position = conf->reshape_progress; ++ mddev->curr_resync_completed = sector_nr; ++ conf->reshape_checkpoint = jiffies; ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ md_wakeup_thread(mddev->thread); ++ wait_event(mddev->sb_wait, ++ !test_bit(MD_CHANGE_DEVS, &mddev->flags) ++ || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); ++ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) ++ goto ret; ++ spin_lock_irq(&conf->device_lock); ++ conf->reshape_safe = mddev->reshape_position; ++ spin_unlock_irq(&conf->device_lock); ++ wake_up(&conf->wait_for_overlap); ++ sysfs_notify(&mddev->kobj, NULL, "sync_completed"); ++ } ++ret: ++ return reshape_sectors; ++} ++ ++static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped) ++{ ++ struct r5conf *conf = mddev->private; ++ struct stripe_head *sh; ++ sector_t max_sector = mddev->dev_sectors; ++ sector_t sync_blocks; ++ int still_degraded = 0; ++ int i; ++ ++ if (sector_nr >= max_sector) { ++ /* just being told to finish up .. nothing much to do */ ++ ++ if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { ++ end_reshape(conf); ++ return 0; ++ } ++ ++ if (mddev->curr_resync < max_sector) /* aborted */ ++ bitmap_end_sync(mddev->bitmap, mddev->curr_resync, ++ &sync_blocks, 1); ++ else /* completed sync */ ++ conf->fullsync = 0; ++ bitmap_close_sync(mddev->bitmap); ++ ++ return 0; ++ } ++ ++ /* Allow raid5_quiesce to complete */ ++ wait_event(conf->wait_for_overlap, conf->quiesce != 2); ++ ++ if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) ++ return reshape_request(mddev, sector_nr, skipped); ++ ++ /* No need to check resync_max as we never do more than one ++ * stripe, and as resync_max will always be on a chunk boundary, ++ * if the check in md_do_sync didn't fire, there is no chance ++ * of overstepping resync_max here ++ */ ++ ++ /* if there is too many failed drives and we are trying ++ * to resync, then assert that we are finished, because there is ++ * nothing we can do. ++ */ ++ if (mddev->degraded >= conf->max_degraded && ++ test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { ++ sector_t rv = mddev->dev_sectors - sector_nr; ++ *skipped = 1; ++ return rv; ++ } ++ if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && ++ !conf->fullsync && ++ !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && ++ sync_blocks >= STRIPE_SECTORS) { ++ /* we can skip this block, and probably more */ ++ sync_blocks /= STRIPE_SECTORS; ++ *skipped = 1; ++ return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ ++ } ++ ++ bitmap_cond_end_sync(mddev->bitmap, sector_nr); ++ ++ sh = get_active_stripe(conf, sector_nr, 0, 1, 0); ++ if (sh == NULL) { ++ sh = get_active_stripe(conf, sector_nr, 0, 0, 0); ++ /* make sure we don't swamp the stripe cache if someone else ++ * is trying to get access ++ */ ++ schedule_timeout_uninterruptible(1); ++ } ++ /* Need to check if array will still be degraded after recovery/resync ++ * Note in case of > 1 drive failures it's possible we're rebuilding ++ * one drive while leaving another faulty drive in array. ++ */ ++ rcu_read_lock(); ++ for (i = 0; i < conf->raid_disks; i++) { ++ struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev); ++ ++ if (rdev == NULL || test_bit(Faulty, &rdev->flags)) ++ still_degraded = 1; ++ } ++ rcu_read_unlock(); ++ ++ bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); ++ ++ set_bit(STRIPE_SYNC_REQUESTED, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ ++ release_stripe(sh); ++ ++ return STRIPE_SECTORS; ++} ++ ++static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) ++{ ++ /* We may not be able to submit a whole bio at once as there ++ * may not be enough stripe_heads available. ++ * We cannot pre-allocate enough stripe_heads as we may need ++ * more than exist in the cache (if we allow ever large chunks). ++ * So we do one stripe head at a time and record in ++ * ->bi_hw_segments how many have been done. ++ * ++ * We *know* that this entire raid_bio is in one chunk, so ++ * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. ++ */ ++ struct stripe_head *sh; ++ int dd_idx; ++ sector_t sector, logical_sector, last_sector; ++ int scnt = 0; ++ int remaining; ++ int handled = 0; ++ ++ logical_sector = raid_bio->bi_iter.bi_sector & ++ ~((sector_t)STRIPE_SECTORS-1); ++ sector = raid5_compute_sector(conf, logical_sector, ++ 0, &dd_idx, NULL); ++ last_sector = bio_end_sector(raid_bio); ++ ++ for (; logical_sector < last_sector; ++ logical_sector += STRIPE_SECTORS, ++ sector += STRIPE_SECTORS, ++ scnt++) { ++ ++ if (scnt < raid5_bi_processed_stripes(raid_bio)) ++ /* already done this stripe */ ++ continue; ++ ++ sh = get_active_stripe(conf, sector, 0, 1, 1); ++ ++ if (!sh) { ++ /* failed to get a stripe - must wait */ ++ raid5_set_bi_processed_stripes(raid_bio, scnt); ++ conf->retry_read_aligned = raid_bio; ++ return handled; ++ } ++ ++ if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) { ++ release_stripe(sh); ++ raid5_set_bi_processed_stripes(raid_bio, scnt); ++ conf->retry_read_aligned = raid_bio; ++ return handled; ++ } ++ ++ set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags); ++ handle_stripe(sh); ++ release_stripe(sh); ++ handled++; ++ } ++ remaining = raid5_dec_bi_active_stripes(raid_bio); ++ if (remaining == 0) { ++ trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), ++ raid_bio, 0); ++ bio_endio(raid_bio, 0); ++ } ++ if (atomic_dec_and_test(&conf->active_aligned_reads)) ++ wake_up(&conf->wait_for_stripe); ++ return handled; ++} ++ ++static int handle_active_stripes(struct r5conf *conf, int group, ++ struct r5worker *worker, ++ struct list_head *temp_inactive_list) ++{ ++ struct stripe_head *batch[MAX_STRIPE_BATCH], *sh; ++ int i, batch_size = 0, hash; ++ bool release_inactive = false; ++ ++ while (batch_size < MAX_STRIPE_BATCH && ++ (sh = __get_priority_stripe(conf, group)) != NULL) ++ batch[batch_size++] = sh; ++ ++ if (batch_size == 0) { ++ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) ++ if (!list_empty(temp_inactive_list + i)) ++ break; ++ if (i == NR_STRIPE_HASH_LOCKS) ++ return batch_size; ++ release_inactive = true; ++ } ++ spin_unlock_irq(&conf->device_lock); ++ ++ release_inactive_stripe_list(conf, temp_inactive_list, ++ NR_STRIPE_HASH_LOCKS); ++ ++ if (release_inactive) { ++ spin_lock_irq(&conf->device_lock); ++ return 0; ++ } ++ ++ for (i = 0; i < batch_size; i++) ++ handle_stripe(batch[i]); ++ ++ cond_resched(); ++ ++ spin_lock_irq(&conf->device_lock); ++ for (i = 0; i < batch_size; i++) { ++ hash = batch[i]->hash_lock_index; ++ __release_stripe(conf, batch[i], &temp_inactive_list[hash]); ++ } ++ return batch_size; ++} ++ ++static void raid5_do_work(struct work_struct *work) ++{ ++ struct r5worker *worker = container_of(work, struct r5worker, work); ++ struct r5worker_group *group = worker->group; ++ struct r5conf *conf = group->conf; ++ int group_id = group - conf->worker_groups; ++ int handled; ++ struct blk_plug plug; ++ ++ pr_debug("+++ raid5worker active\n"); ++ ++ blk_start_plug(&plug); ++ handled = 0; ++ spin_lock_irq(&conf->device_lock); ++ while (1) { ++ int batch_size, released; ++ ++ released = release_stripe_list(conf, worker->temp_inactive_list); ++ ++ batch_size = handle_active_stripes(conf, group_id, worker, ++ worker->temp_inactive_list); ++ worker->working = false; ++ if (!batch_size && !released) ++ break; ++ handled += batch_size; ++ } ++ pr_debug("%d stripes handled\n", handled); ++ ++ spin_unlock_irq(&conf->device_lock); ++ blk_finish_plug(&plug); ++ ++ pr_debug("--- raid5worker inactive\n"); ++} ++ ++/* ++ * This is our raid5 kernel thread. ++ * ++ * We scan the hash table for stripes which can be handled now. ++ * During the scan, completed stripes are saved for us by the interrupt ++ * handler, so that they will not have to wait for our next wakeup. ++ */ ++static void raid5d(struct md_thread *thread) ++{ ++ struct mddev *mddev = thread->mddev; ++ struct r5conf *conf = mddev->private; ++ int handled; ++ struct blk_plug plug; ++ ++ pr_debug("+++ raid5d active\n"); ++ ++ md_check_recovery(mddev); ++ ++ blk_start_plug(&plug); ++ handled = 0; ++ spin_lock_irq(&conf->device_lock); ++ while (1) { ++ struct bio *bio; ++ int batch_size, released; ++ ++ released = release_stripe_list(conf, conf->temp_inactive_list); ++ if (released) ++ clear_bit(R5_DID_ALLOC, &conf->cache_state); ++ ++ if ( ++ !list_empty(&conf->bitmap_list)) { ++ /* Now is a good time to flush some bitmap updates */ ++ conf->seq_flush++; ++ spin_unlock_irq(&conf->device_lock); ++ bitmap_unplug(mddev->bitmap); ++ spin_lock_irq(&conf->device_lock); ++ conf->seq_write = conf->seq_flush; ++ activate_bit_delay(conf, conf->temp_inactive_list); ++ } ++ raid5_activate_delayed(conf); ++ ++ while ((bio = remove_bio_from_retry(conf))) { ++ int ok; ++ spin_unlock_irq(&conf->device_lock); ++ ok = retry_aligned_read(conf, bio); ++ spin_lock_irq(&conf->device_lock); ++ if (!ok) ++ break; ++ handled++; ++ } ++ ++ batch_size = handle_active_stripes(conf, ANY_GROUP, NULL, ++ conf->temp_inactive_list); ++ if (!batch_size && !released) ++ break; ++ handled += batch_size; ++ ++ if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) { ++ spin_unlock_irq(&conf->device_lock); ++ md_check_recovery(mddev); ++ spin_lock_irq(&conf->device_lock); ++ } ++ } ++ pr_debug("%d stripes handled\n", handled); ++ ++ spin_unlock_irq(&conf->device_lock); ++ if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) && ++ mutex_trylock(&conf->cache_size_mutex)) { ++ grow_one_stripe(conf, __GFP_NOWARN); ++ /* Set flag even if allocation failed. This helps ++ * slow down allocation requests when mem is short ++ */ ++ set_bit(R5_DID_ALLOC, &conf->cache_state); ++ mutex_unlock(&conf->cache_size_mutex); ++ } ++ ++ async_tx_issue_pending_all(); ++ blk_finish_plug(&plug); ++ ++ pr_debug("--- raid5d inactive\n"); ++} ++ ++static ssize_t ++raid5_show_stripe_cache_size(struct mddev *mddev, char *page) ++{ ++ struct r5conf *conf; ++ int ret = 0; ++ spin_lock(&mddev->lock); ++ conf = mddev->private; ++ if (conf) ++ ret = sprintf(page, "%d\n", conf->min_nr_stripes); ++ spin_unlock(&mddev->lock); ++ return ret; ++} ++ ++int ++raid5_set_cache_size(struct mddev *mddev, int size) ++{ ++ struct r5conf *conf = mddev->private; ++ int err; ++ ++ if (size <= 16 || size > 32768) ++ return -EINVAL; ++ ++ conf->min_nr_stripes = size; ++ mutex_lock(&conf->cache_size_mutex); ++ while (size < conf->max_nr_stripes && ++ drop_one_stripe(conf)) ++ ; ++ mutex_unlock(&conf->cache_size_mutex); ++ ++ ++ err = md_allow_write(mddev); ++ if (err) ++ return err; ++ ++ mutex_lock(&conf->cache_size_mutex); ++ while (size > conf->max_nr_stripes) ++ if (!grow_one_stripe(conf, GFP_KERNEL)) ++ break; ++ mutex_unlock(&conf->cache_size_mutex); ++ ++ return 0; ++} ++EXPORT_SYMBOL(raid5_set_cache_size); ++ ++static ssize_t ++raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len) ++{ ++ struct r5conf *conf; ++ unsigned long new; ++ int err; ++ ++ if (len >= PAGE_SIZE) ++ return -EINVAL; ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; ++ err = mddev_lock(mddev); ++ if (err) ++ return err; ++ conf = mddev->private; ++ if (!conf) ++ err = -ENODEV; ++ else ++ err = raid5_set_cache_size(mddev, new); ++ mddev_unlock(mddev); ++ ++ return err ?: len; ++} ++ ++static struct md_sysfs_entry ++raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, ++ raid5_show_stripe_cache_size, ++ raid5_store_stripe_cache_size); ++ ++static ssize_t ++raid5_show_rmw_level(struct mddev *mddev, char *page) ++{ ++ struct r5conf *conf = mddev->private; ++ if (conf) ++ return sprintf(page, "%d\n", conf->rmw_level); ++ else ++ return 0; ++} ++ ++static ssize_t ++raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len) ++{ ++ struct r5conf *conf = mddev->private; ++ unsigned long new; ++ ++ if (!conf) ++ return -ENODEV; ++ ++ if (len >= PAGE_SIZE) ++ return -EINVAL; ++ ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; ++ ++ if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome) ++ return -EINVAL; ++ ++ if (new != PARITY_DISABLE_RMW && ++ new != PARITY_ENABLE_RMW && ++ new != PARITY_PREFER_RMW) ++ return -EINVAL; ++ ++ conf->rmw_level = new; ++ return len; ++} ++ ++static struct md_sysfs_entry ++raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR, ++ raid5_show_rmw_level, ++ raid5_store_rmw_level); ++ ++ ++static ssize_t ++raid5_show_preread_threshold(struct mddev *mddev, char *page) ++{ ++ struct r5conf *conf; ++ int ret = 0; ++ spin_lock(&mddev->lock); ++ conf = mddev->private; ++ if (conf) ++ ret = sprintf(page, "%d\n", conf->bypass_threshold); ++ spin_unlock(&mddev->lock); ++ return ret; ++} ++ ++static ssize_t ++raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len) ++{ ++ struct r5conf *conf; ++ unsigned long new; ++ int err; ++ ++ if (len >= PAGE_SIZE) ++ return -EINVAL; ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; ++ ++ err = mddev_lock(mddev); ++ if (err) ++ return err; ++ conf = mddev->private; ++ if (!conf) ++ err = -ENODEV; ++ else if (new > conf->min_nr_stripes) ++ err = -EINVAL; ++ else ++ conf->bypass_threshold = new; ++ mddev_unlock(mddev); ++ return err ?: len; ++} ++ ++static struct md_sysfs_entry ++raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, ++ S_IRUGO | S_IWUSR, ++ raid5_show_preread_threshold, ++ raid5_store_preread_threshold); ++ ++static ssize_t ++raid5_show_skip_copy(struct mddev *mddev, char *page) ++{ ++ struct r5conf *conf; ++ int ret = 0; ++ spin_lock(&mddev->lock); ++ conf = mddev->private; ++ if (conf) ++ ret = sprintf(page, "%d\n", conf->skip_copy); ++ spin_unlock(&mddev->lock); ++ return ret; ++} ++ ++static ssize_t ++raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len) ++{ ++ struct r5conf *conf; ++ unsigned long new; ++ int err; ++ ++ if (len >= PAGE_SIZE) ++ return -EINVAL; ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; ++ new = !!new; ++ ++ err = mddev_lock(mddev); ++ if (err) ++ return err; ++ conf = mddev->private; ++ if (!conf) ++ err = -ENODEV; ++ else if (new != conf->skip_copy) { ++ mddev_suspend(mddev); ++ conf->skip_copy = new; ++ if (new) ++ mddev->queue->backing_dev_info.capabilities |= ++ BDI_CAP_STABLE_WRITES; ++ else ++ mddev->queue->backing_dev_info.capabilities &= ++ ~BDI_CAP_STABLE_WRITES; ++ mddev_resume(mddev); ++ } ++ mddev_unlock(mddev); ++ return err ?: len; ++} ++ ++static struct md_sysfs_entry ++raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR, ++ raid5_show_skip_copy, ++ raid5_store_skip_copy); ++ ++static ssize_t ++stripe_cache_active_show(struct mddev *mddev, char *page) ++{ ++ struct r5conf *conf = mddev->private; ++ if (conf) ++ return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); ++ else ++ return 0; ++} ++ ++static struct md_sysfs_entry ++raid5_stripecache_active = __ATTR_RO(stripe_cache_active); ++ ++static ssize_t ++raid5_show_group_thread_cnt(struct mddev *mddev, char *page) ++{ ++ struct r5conf *conf; ++ int ret = 0; ++ spin_lock(&mddev->lock); ++ conf = mddev->private; ++ if (conf) ++ ret = sprintf(page, "%d\n", conf->worker_cnt_per_group); ++ spin_unlock(&mddev->lock); ++ return ret; ++} ++ ++static int alloc_thread_groups(struct r5conf *conf, int cnt, ++ int *group_cnt, ++ int *worker_cnt_per_group, ++ struct r5worker_group **worker_groups); ++static ssize_t ++raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) ++{ ++ struct r5conf *conf; ++ unsigned long new; ++ int err; ++ struct r5worker_group *new_groups, *old_groups; ++ int group_cnt, worker_cnt_per_group; ++ ++ if (len >= PAGE_SIZE) ++ return -EINVAL; ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; ++ ++ err = mddev_lock(mddev); ++ if (err) ++ return err; ++ conf = mddev->private; ++ if (!conf) ++ err = -ENODEV; ++ else if (new != conf->worker_cnt_per_group) { ++ mddev_suspend(mddev); ++ ++ old_groups = conf->worker_groups; ++ if (old_groups) ++ flush_workqueue(raid5_wq); ++ ++ err = alloc_thread_groups(conf, new, ++ &group_cnt, &worker_cnt_per_group, ++ &new_groups); ++ if (!err) { ++ spin_lock_irq(&conf->device_lock); ++ conf->group_cnt = group_cnt; ++ conf->worker_cnt_per_group = worker_cnt_per_group; ++ conf->worker_groups = new_groups; ++ spin_unlock_irq(&conf->device_lock); ++ ++ if (old_groups) ++ kfree(old_groups[0].workers); ++ kfree(old_groups); ++ } ++ mddev_resume(mddev); ++ } ++ mddev_unlock(mddev); ++ ++ return err ?: len; ++} ++ ++static struct md_sysfs_entry ++raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR, ++ raid5_show_group_thread_cnt, ++ raid5_store_group_thread_cnt); ++ ++static struct attribute *raid5_attrs[] = { ++ &raid5_stripecache_size.attr, ++ &raid5_stripecache_active.attr, ++ &raid5_preread_bypass_threshold.attr, ++ &raid5_group_thread_cnt.attr, ++ &raid5_skip_copy.attr, ++ &raid5_rmw_level.attr, ++ NULL, ++}; ++static struct attribute_group raid5_attrs_group = { ++ .name = NULL, ++ .attrs = raid5_attrs, ++}; ++ ++static int alloc_thread_groups(struct r5conf *conf, int cnt, ++ int *group_cnt, ++ int *worker_cnt_per_group, ++ struct r5worker_group **worker_groups) ++{ ++ int i, j, k; ++ ssize_t size; ++ struct r5worker *workers; ++ ++ *worker_cnt_per_group = cnt; ++ if (cnt == 0) { ++ *group_cnt = 0; ++ *worker_groups = NULL; ++ return 0; ++ } ++ *group_cnt = num_possible_nodes(); ++ size = sizeof(struct r5worker) * cnt; ++ workers = kzalloc(size * *group_cnt, GFP_NOIO); ++ *worker_groups = kzalloc(sizeof(struct r5worker_group) * ++ *group_cnt, GFP_NOIO); ++ if (!*worker_groups || !workers) { ++ kfree(workers); ++ kfree(*worker_groups); ++ return -ENOMEM; ++ } ++ ++ for (i = 0; i < *group_cnt; i++) { ++ struct r5worker_group *group; ++ ++ group = &(*worker_groups)[i]; ++ INIT_LIST_HEAD(&group->handle_list); ++ group->conf = conf; ++ group->workers = workers + i * cnt; ++ ++ for (j = 0; j < cnt; j++) { ++ struct r5worker *worker = group->workers + j; ++ worker->group = group; ++ INIT_WORK(&worker->work, raid5_do_work); ++ ++ for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++) ++ INIT_LIST_HEAD(worker->temp_inactive_list + k); ++ } ++ } ++ ++ return 0; ++} ++ ++static void free_thread_groups(struct r5conf *conf) ++{ ++ if (conf->worker_groups) ++ kfree(conf->worker_groups[0].workers); ++ kfree(conf->worker_groups); ++ conf->worker_groups = NULL; ++} ++ ++static sector_t ++raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) ++{ ++ struct r5conf *conf = mddev->private; ++ ++ if (!sectors) ++ sectors = mddev->dev_sectors; ++ if (!raid_disks) ++ /* size is defined by the smallest of previous and new size */ ++ raid_disks = min(conf->raid_disks, conf->previous_raid_disks); ++ ++ sectors &= ~((sector_t)mddev->chunk_sectors - 1); ++ sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); ++ return sectors * (raid_disks - conf->max_degraded); ++} ++ ++static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) ++{ ++ safe_put_page(percpu->spare_page); ++ if (percpu->scribble) ++ flex_array_free(percpu->scribble); ++ percpu->spare_page = NULL; ++ percpu->scribble = NULL; ++} ++ ++static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) ++{ ++ if (conf->level == 6 && !percpu->spare_page) ++ percpu->spare_page = alloc_page(GFP_KERNEL); ++ if (!percpu->scribble) ++ percpu->scribble = scribble_alloc(max(conf->raid_disks, ++ conf->previous_raid_disks), ++ max(conf->chunk_sectors, ++ conf->prev_chunk_sectors) ++ / STRIPE_SECTORS, ++ GFP_KERNEL); ++ ++ if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) { ++ free_scratch_buffer(conf, percpu); ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++static void raid5_free_percpu(struct r5conf *conf) ++{ ++ unsigned long cpu; ++ ++ if (!conf->percpu) ++ return; ++ ++#ifdef CONFIG_HOTPLUG_CPU ++ unregister_cpu_notifier(&conf->cpu_notify); ++#endif ++ ++ get_online_cpus(); ++ for_each_possible_cpu(cpu) ++ free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); ++ put_online_cpus(); ++ ++ free_percpu(conf->percpu); ++} ++ ++static void free_conf(struct r5conf *conf) ++{ ++ if (conf->shrinker.seeks) ++ unregister_shrinker(&conf->shrinker); ++ free_thread_groups(conf); ++ shrink_stripes(conf); ++ raid5_free_percpu(conf); ++ kfree(conf->disks); ++ kfree(conf->stripe_hashtbl); ++ kfree(conf); ++} ++ ++#ifdef CONFIG_HOTPLUG_CPU ++static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, ++ void *hcpu) ++{ ++ struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify); ++ long cpu = (long)hcpu; ++ struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); ++ ++ switch (action) { ++ case CPU_UP_PREPARE: ++ case CPU_UP_PREPARE_FROZEN: ++ if (alloc_scratch_buffer(conf, percpu)) { ++ pr_err("%s: failed memory allocation for cpu%ld\n", ++ __func__, cpu); ++ return notifier_from_errno(-ENOMEM); ++ } ++ break; ++ case CPU_DEAD: ++ case CPU_DEAD_FROZEN: ++ free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); ++ break; ++ default: ++ break; ++ } ++ return NOTIFY_OK; ++} ++#endif ++ ++static int raid5_alloc_percpu(struct r5conf *conf) ++{ ++ unsigned long cpu; ++ int err = 0; ++ ++ conf->percpu = alloc_percpu(struct raid5_percpu); ++ if (!conf->percpu) ++ return -ENOMEM; ++ ++#ifdef CONFIG_HOTPLUG_CPU ++ conf->cpu_notify.notifier_call = raid456_cpu_notify; ++ conf->cpu_notify.priority = 0; ++ err = register_cpu_notifier(&conf->cpu_notify); ++ if (err) ++ return err; ++#endif ++ ++ get_online_cpus(); ++ for_each_present_cpu(cpu) { ++ err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); ++ if (err) { ++ pr_err("%s: failed memory allocation for cpu%ld\n", ++ __func__, cpu); ++ break; ++ } ++ } ++ put_online_cpus(); ++ ++ return err; ++} ++ ++static unsigned long raid5_cache_scan(struct shrinker *shrink, ++ struct shrink_control *sc) ++{ ++ struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); ++ unsigned long ret = SHRINK_STOP; ++ ++ if (mutex_trylock(&conf->cache_size_mutex)) { ++ ret= 0; ++ while (ret < sc->nr_to_scan && ++ conf->max_nr_stripes > conf->min_nr_stripes) { ++ if (drop_one_stripe(conf) == 0) { ++ ret = SHRINK_STOP; ++ break; ++ } ++ ret++; ++ } ++ mutex_unlock(&conf->cache_size_mutex); ++ } ++ return ret; ++} ++ ++static unsigned long raid5_cache_count(struct shrinker *shrink, ++ struct shrink_control *sc) ++{ ++ struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); ++ ++ if (conf->max_nr_stripes < conf->min_nr_stripes) ++ /* unlikely, but not impossible */ ++ return 0; ++ return conf->max_nr_stripes - conf->min_nr_stripes; ++} ++ ++static struct r5conf *setup_conf(struct mddev *mddev) ++{ ++ struct r5conf *conf; ++ int raid_disk, memory, max_disks; ++ struct md_rdev *rdev; ++ struct disk_info *disk; ++ char pers_name[6]; ++ int i; ++ int group_cnt, worker_cnt_per_group; ++ struct r5worker_group *new_group; ++ ++ if (mddev->new_level != 5 ++ && mddev->new_level != 4 ++ && mddev->new_level != 6) { ++ printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n", ++ mdname(mddev), mddev->new_level); ++ return ERR_PTR(-EIO); ++ } ++ if ((mddev->new_level == 5 ++ && !algorithm_valid_raid5(mddev->new_layout)) || ++ (mddev->new_level == 6 ++ && !algorithm_valid_raid6(mddev->new_layout))) { ++ printk(KERN_ERR "md/raid:%s: layout %d not supported\n", ++ mdname(mddev), mddev->new_layout); ++ return ERR_PTR(-EIO); ++ } ++ if (mddev->new_level == 6 && mddev->raid_disks < 4) { ++ printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n", ++ mdname(mddev), mddev->raid_disks); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ if (!mddev->new_chunk_sectors || ++ (mddev->new_chunk_sectors << 9) % PAGE_SIZE || ++ !is_power_of_2(mddev->new_chunk_sectors)) { ++ printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n", ++ mdname(mddev), mddev->new_chunk_sectors << 9); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); ++ if (conf == NULL) ++ goto abort; ++ /* Don't enable multi-threading by default*/ ++ if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group, ++ &new_group)) { ++ conf->group_cnt = group_cnt; ++ conf->worker_cnt_per_group = worker_cnt_per_group; ++ conf->worker_groups = new_group; ++ } else ++ goto abort; ++ spin_lock_init(&conf->device_lock); ++ seqcount_init(&conf->gen_lock); ++ mutex_init(&conf->cache_size_mutex); ++ init_waitqueue_head(&conf->wait_for_stripe); ++ init_waitqueue_head(&conf->wait_for_overlap); ++ INIT_LIST_HEAD(&conf->handle_list); ++ INIT_LIST_HEAD(&conf->hold_list); ++ INIT_LIST_HEAD(&conf->delayed_list); ++ INIT_LIST_HEAD(&conf->bitmap_list); ++ init_llist_head(&conf->released_stripes); ++ atomic_set(&conf->active_stripes, 0); ++ atomic_set(&conf->preread_active_stripes, 0); ++ atomic_set(&conf->active_aligned_reads, 0); ++ conf->bypass_threshold = BYPASS_THRESHOLD; ++ conf->recovery_disabled = mddev->recovery_disabled - 1; ++ ++ conf->raid_disks = mddev->raid_disks; ++ if (mddev->reshape_position == MaxSector) ++ conf->previous_raid_disks = mddev->raid_disks; ++ else ++ conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; ++ max_disks = max(conf->raid_disks, conf->previous_raid_disks); ++ ++ conf->disks = kzalloc(max_disks * sizeof(struct disk_info), ++ GFP_KERNEL); ++ if (!conf->disks) ++ goto abort; ++ ++ conf->mddev = mddev; ++ ++ if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) ++ goto abort; ++ ++ /* We init hash_locks[0] separately to that it can be used ++ * as the reference lock in the spin_lock_nest_lock() call ++ * in lock_all_device_hash_locks_irq in order to convince ++ * lockdep that we know what we are doing. ++ */ ++ spin_lock_init(conf->hash_locks); ++ for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) ++ spin_lock_init(conf->hash_locks + i); ++ ++ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) ++ INIT_LIST_HEAD(conf->inactive_list + i); ++ ++ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) ++ INIT_LIST_HEAD(conf->temp_inactive_list + i); ++ ++ conf->level = mddev->new_level; ++ conf->chunk_sectors = mddev->new_chunk_sectors; ++ if (raid5_alloc_percpu(conf) != 0) ++ goto abort; ++ ++ pr_debug("raid456: run(%s) called.\n", mdname(mddev)); ++ ++ rdev_for_each(rdev, mddev) { ++ raid_disk = rdev->raid_disk; ++ if (raid_disk >= max_disks ++ || raid_disk < 0) ++ continue; ++ disk = conf->disks + raid_disk; ++ ++ if (test_bit(Replacement, &rdev->flags)) { ++ if (disk->replacement) ++ goto abort; ++ disk->replacement = rdev; ++ } else { ++ if (disk->rdev) ++ goto abort; ++ disk->rdev = rdev; ++ } ++ ++ if (test_bit(In_sync, &rdev->flags)) { ++ char b[BDEVNAME_SIZE]; ++ printk(KERN_INFO "md/raid:%s: device %s operational as raid" ++ " disk %d\n", ++ mdname(mddev), bdevname(rdev->bdev, b), raid_disk); ++ } else if (rdev->saved_raid_disk != raid_disk) ++ /* Cannot rely on bitmap to complete recovery */ ++ conf->fullsync = 1; ++ } ++ ++ conf->level = mddev->new_level; ++ if (conf->level == 6) { ++ conf->max_degraded = 2; ++ if (raid6_call.xor_syndrome) ++ conf->rmw_level = PARITY_ENABLE_RMW; ++ else ++ conf->rmw_level = PARITY_DISABLE_RMW; ++ } else { ++ conf->max_degraded = 1; ++ conf->rmw_level = PARITY_ENABLE_RMW; ++ } ++ conf->algorithm = mddev->new_layout; ++ conf->reshape_progress = mddev->reshape_position; ++ if (conf->reshape_progress != MaxSector) { ++ conf->prev_chunk_sectors = mddev->chunk_sectors; ++ conf->prev_algo = mddev->layout; ++ } ++ ++ conf->min_nr_stripes = NR_STRIPES; ++ memory = conf->min_nr_stripes * (sizeof(struct stripe_head) + ++ max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; ++ atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS); ++ if (grow_stripes(conf, conf->min_nr_stripes)) { ++ printk(KERN_ERR ++ "md/raid:%s: couldn't allocate %dkB for buffers\n", ++ mdname(mddev), memory); ++ goto abort; ++ } else ++ printk(KERN_INFO "md/raid:%s: allocated %dkB\n", ++ mdname(mddev), memory); ++ /* ++ * Losing a stripe head costs more than the time to refill it, ++ * it reduces the queue depth and so can hurt throughput. ++ * So set it rather large, scaled by number of devices. ++ */ ++ conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4; ++ conf->shrinker.scan_objects = raid5_cache_scan; ++ conf->shrinker.count_objects = raid5_cache_count; ++ conf->shrinker.batch = 128; ++ conf->shrinker.flags = 0; ++ register_shrinker(&conf->shrinker); ++ ++ sprintf(pers_name, "raid%d", mddev->new_level); ++ conf->thread = md_register_thread(raid5d, mddev, pers_name); ++ if (!conf->thread) { ++ printk(KERN_ERR ++ "md/raid:%s: couldn't allocate thread.\n", ++ mdname(mddev)); ++ goto abort; ++ } ++ ++ return conf; ++ ++ abort: ++ if (conf) { ++ free_conf(conf); ++ return ERR_PTR(-EIO); ++ } else ++ return ERR_PTR(-ENOMEM); ++} ++ ++static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) ++{ ++ switch (algo) { ++ case ALGORITHM_PARITY_0: ++ if (raid_disk < max_degraded) ++ return 1; ++ break; ++ case ALGORITHM_PARITY_N: ++ if (raid_disk >= raid_disks - max_degraded) ++ return 1; ++ break; ++ case ALGORITHM_PARITY_0_6: ++ if (raid_disk == 0 || ++ raid_disk == raid_disks - 1) ++ return 1; ++ break; ++ case ALGORITHM_LEFT_ASYMMETRIC_6: ++ case ALGORITHM_RIGHT_ASYMMETRIC_6: ++ case ALGORITHM_LEFT_SYMMETRIC_6: ++ case ALGORITHM_RIGHT_SYMMETRIC_6: ++ if (raid_disk == raid_disks - 1) ++ return 1; ++ } ++ return 0; ++} ++ ++static int run(struct mddev *mddev) ++{ ++ struct r5conf *conf; ++ int working_disks = 0; ++ int dirty_parity_disks = 0; ++ struct md_rdev *rdev; ++ sector_t reshape_offset = 0; ++ int i; ++ long long min_offset_diff = 0; ++ int first = 1; ++ ++ if (mddev->recovery_cp != MaxSector) ++ printk(KERN_NOTICE "md/raid:%s: not clean" ++ " -- starting background reconstruction\n", ++ mdname(mddev)); ++ ++ rdev_for_each(rdev, mddev) { ++ long long diff; ++ if (rdev->raid_disk < 0) ++ continue; ++ diff = (rdev->new_data_offset - rdev->data_offset); ++ if (first) { ++ min_offset_diff = diff; ++ first = 0; ++ } else if (mddev->reshape_backwards && ++ diff < min_offset_diff) ++ min_offset_diff = diff; ++ else if (!mddev->reshape_backwards && ++ diff > min_offset_diff) ++ min_offset_diff = diff; ++ } ++ ++ if (mddev->reshape_position != MaxSector) { ++ /* Check that we can continue the reshape. ++ * Difficulties arise if the stripe we would write to ++ * next is at or after the stripe we would read from next. ++ * For a reshape that changes the number of devices, this ++ * is only possible for a very short time, and mdadm makes ++ * sure that time appears to have past before assembling ++ * the array. So we fail if that time hasn't passed. ++ * For a reshape that keeps the number of devices the same ++ * mdadm must be monitoring the reshape can keeping the ++ * critical areas read-only and backed up. It will start ++ * the array in read-only mode, so we check for that. ++ */ ++ sector_t here_new, here_old; ++ int old_disks; ++ int max_degraded = (mddev->level == 6 ? 2 : 1); ++ ++ if (mddev->new_level != mddev->level) { ++ printk(KERN_ERR "md/raid:%s: unsupported reshape " ++ "required - aborting.\n", ++ mdname(mddev)); ++ return -EINVAL; ++ } ++ old_disks = mddev->raid_disks - mddev->delta_disks; ++ /* reshape_position must be on a new-stripe boundary, and one ++ * further up in new geometry must map after here in old ++ * geometry. ++ */ ++ here_new = mddev->reshape_position; ++ if (sector_div(here_new, mddev->new_chunk_sectors * ++ (mddev->raid_disks - max_degraded))) { ++ printk(KERN_ERR "md/raid:%s: reshape_position not " ++ "on a stripe boundary\n", mdname(mddev)); ++ return -EINVAL; ++ } ++ reshape_offset = here_new * mddev->new_chunk_sectors; ++ /* here_new is the stripe we will write to */ ++ here_old = mddev->reshape_position; ++ sector_div(here_old, mddev->chunk_sectors * ++ (old_disks-max_degraded)); ++ /* here_old is the first stripe that we might need to read ++ * from */ ++ if (mddev->delta_disks == 0) { ++ if ((here_new * mddev->new_chunk_sectors != ++ here_old * mddev->chunk_sectors)) { ++ printk(KERN_ERR "md/raid:%s: reshape position is" ++ " confused - aborting\n", mdname(mddev)); ++ return -EINVAL; ++ } ++ /* We cannot be sure it is safe to start an in-place ++ * reshape. It is only safe if user-space is monitoring ++ * and taking constant backups. ++ * mdadm always starts a situation like this in ++ * readonly mode so it can take control before ++ * allowing any writes. So just check for that. ++ */ ++ if (abs(min_offset_diff) >= mddev->chunk_sectors && ++ abs(min_offset_diff) >= mddev->new_chunk_sectors) ++ /* not really in-place - so OK */; ++ else if (mddev->ro == 0) { ++ printk(KERN_ERR "md/raid:%s: in-place reshape " ++ "must be started in read-only mode " ++ "- aborting\n", ++ mdname(mddev)); ++ return -EINVAL; ++ } ++ } else if (mddev->reshape_backwards ++ ? (here_new * mddev->new_chunk_sectors + min_offset_diff <= ++ here_old * mddev->chunk_sectors) ++ : (here_new * mddev->new_chunk_sectors >= ++ here_old * mddev->chunk_sectors + (-min_offset_diff))) { ++ /* Reading from the same stripe as writing to - bad */ ++ printk(KERN_ERR "md/raid:%s: reshape_position too early for " ++ "auto-recovery - aborting.\n", ++ mdname(mddev)); ++ return -EINVAL; ++ } ++ printk(KERN_INFO "md/raid:%s: reshape will continue\n", ++ mdname(mddev)); ++ /* OK, we should be able to continue; */ ++ } else { ++ BUG_ON(mddev->level != mddev->new_level); ++ BUG_ON(mddev->layout != mddev->new_layout); ++ BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); ++ BUG_ON(mddev->delta_disks != 0); ++ } ++ ++ if (mddev->private == NULL) ++ conf = setup_conf(mddev); ++ else ++ conf = mddev->private; ++ ++ if (IS_ERR(conf)) ++ return PTR_ERR(conf); ++ ++ conf->min_offset_diff = min_offset_diff; ++ mddev->thread = conf->thread; ++ conf->thread = NULL; ++ mddev->private = conf; ++ ++ for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; ++ i++) { ++ rdev = conf->disks[i].rdev; ++ if (!rdev && conf->disks[i].replacement) { ++ /* The replacement is all we have yet */ ++ rdev = conf->disks[i].replacement; ++ conf->disks[i].replacement = NULL; ++ clear_bit(Replacement, &rdev->flags); ++ conf->disks[i].rdev = rdev; ++ } ++ if (!rdev) ++ continue; ++ if (conf->disks[i].replacement && ++ conf->reshape_progress != MaxSector) { ++ /* replacements and reshape simply do not mix. */ ++ printk(KERN_ERR "md: cannot handle concurrent " ++ "replacement and reshape.\n"); ++ goto abort; ++ } ++ if (test_bit(In_sync, &rdev->flags)) { ++ working_disks++; ++ continue; ++ } ++ /* This disc is not fully in-sync. However if it ++ * just stored parity (beyond the recovery_offset), ++ * when we don't need to be concerned about the ++ * array being dirty. ++ * When reshape goes 'backwards', we never have ++ * partially completed devices, so we only need ++ * to worry about reshape going forwards. ++ */ ++ /* Hack because v0.91 doesn't store recovery_offset properly. */ ++ if (mddev->major_version == 0 && ++ mddev->minor_version > 90) ++ rdev->recovery_offset = reshape_offset; ++ ++ if (rdev->recovery_offset < reshape_offset) { ++ /* We need to check old and new layout */ ++ if (!only_parity(rdev->raid_disk, ++ conf->algorithm, ++ conf->raid_disks, ++ conf->max_degraded)) ++ continue; ++ } ++ if (!only_parity(rdev->raid_disk, ++ conf->prev_algo, ++ conf->previous_raid_disks, ++ conf->max_degraded)) ++ continue; ++ dirty_parity_disks++; ++ } ++ ++ /* ++ * 0 for a fully functional array, 1 or 2 for a degraded array. ++ */ ++ mddev->degraded = calc_degraded(conf); ++ ++ if (has_failed(conf)) { ++ printk(KERN_ERR "md/raid:%s: not enough operational devices" ++ " (%d/%d failed)\n", ++ mdname(mddev), mddev->degraded, conf->raid_disks); ++ goto abort; ++ } ++ ++ /* device size must be a multiple of chunk size */ ++ mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); ++ mddev->resync_max_sectors = mddev->dev_sectors; ++ ++ if (mddev->degraded > dirty_parity_disks && ++ mddev->recovery_cp != MaxSector) { ++ if (mddev->ok_start_degraded) ++ printk(KERN_WARNING ++ "md/raid:%s: starting dirty degraded array" ++ " - data corruption possible.\n", ++ mdname(mddev)); ++ else { ++ printk(KERN_ERR ++ "md/raid:%s: cannot start dirty degraded array.\n", ++ mdname(mddev)); ++ goto abort; ++ } ++ } ++ ++ if (mddev->degraded == 0) ++ printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d" ++ " devices, algorithm %d\n", mdname(mddev), conf->level, ++ mddev->raid_disks-mddev->degraded, mddev->raid_disks, ++ mddev->new_layout); ++ else ++ printk(KERN_ALERT "md/raid:%s: raid level %d active with %d" ++ " out of %d devices, algorithm %d\n", ++ mdname(mddev), conf->level, ++ mddev->raid_disks - mddev->degraded, ++ mddev->raid_disks, mddev->new_layout); ++ ++ print_raid5_conf(conf); ++ ++ if (conf->reshape_progress != MaxSector) { ++ conf->reshape_safe = conf->reshape_progress; ++ atomic_set(&conf->reshape_stripes, 0); ++ clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); ++ clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); ++ set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); ++ set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); ++ mddev->sync_thread = md_register_thread(md_do_sync, mddev, ++ "reshape"); ++ } ++ ++ /* Ok, everything is just fine now */ ++ if (mddev->to_remove == &raid5_attrs_group) ++ mddev->to_remove = NULL; ++ else if (mddev->kobj.sd && ++ sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) ++ printk(KERN_WARNING ++ "raid5: failed to create sysfs attributes for %s\n", ++ mdname(mddev)); ++ md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); ++ ++ if (mddev->queue) { ++ int chunk_size; ++ bool discard_supported = true; ++ /* read-ahead size must cover two whole stripes, which ++ * is 2 * (datadisks) * chunksize where 'n' is the ++ * number of raid devices ++ */ ++ int data_disks = conf->previous_raid_disks - conf->max_degraded; ++ int stripe = data_disks * ++ ((mddev->chunk_sectors << 9) / PAGE_SIZE); ++ if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) ++ mddev->queue->backing_dev_info.ra_pages = 2 * stripe; ++ ++ chunk_size = mddev->chunk_sectors << 9; ++ blk_queue_io_min(mddev->queue, chunk_size); ++ blk_queue_io_opt(mddev->queue, chunk_size * ++ (conf->raid_disks - conf->max_degraded)); ++ mddev->queue->limits.raid_partial_stripes_expensive = 1; ++ /* ++ * We can only discard a whole stripe. It doesn't make sense to ++ * discard data disk but write parity disk ++ */ ++ stripe = stripe * PAGE_SIZE; ++ /* Round up to power of 2, as discard handling ++ * currently assumes that */ ++ while ((stripe-1) & stripe) ++ stripe = (stripe | (stripe-1)) + 1; ++ mddev->queue->limits.discard_alignment = stripe; ++ mddev->queue->limits.discard_granularity = stripe; ++ /* ++ * unaligned part of discard request will be ignored, so can't ++ * guarantee discard_zeroes_data ++ */ ++ mddev->queue->limits.discard_zeroes_data = 0; ++ ++ blk_queue_max_write_same_sectors(mddev->queue, 0); ++ ++ rdev_for_each(rdev, mddev) { ++ disk_stack_limits(mddev->gendisk, rdev->bdev, ++ rdev->data_offset << 9); ++ disk_stack_limits(mddev->gendisk, rdev->bdev, ++ rdev->new_data_offset << 9); ++ /* ++ * discard_zeroes_data is required, otherwise data ++ * could be lost. Consider a scenario: discard a stripe ++ * (the stripe could be inconsistent if ++ * discard_zeroes_data is 0); write one disk of the ++ * stripe (the stripe could be inconsistent again ++ * depending on which disks are used to calculate ++ * parity); the disk is broken; The stripe data of this ++ * disk is lost. ++ */ ++ if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) || ++ !bdev_get_queue(rdev->bdev)-> ++ limits.discard_zeroes_data) ++ discard_supported = false; ++ /* Unfortunately, discard_zeroes_data is not currently ++ * a guarantee - just a hint. So we only allow DISCARD ++ * if the sysadmin has confirmed that only safe devices ++ * are in use by setting a module parameter. ++ */ ++ if (!devices_handle_discard_safely) { ++ if (discard_supported) { ++ pr_info("md/raid456: discard support disabled due to uncertainty.\n"); ++ pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n"); ++ } ++ discard_supported = false; ++ } ++ } ++ ++ if (discard_supported && ++ mddev->queue->limits.max_discard_sectors >= stripe && ++ mddev->queue->limits.discard_granularity >= stripe) ++ queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ++ mddev->queue); ++ else ++ queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, ++ mddev->queue); ++ } ++ ++ return 0; ++abort: ++ md_unregister_thread(&mddev->thread); ++ print_raid5_conf(conf); ++ free_conf(conf); ++ mddev->private = NULL; ++ printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev)); ++ return -EIO; ++} ++ ++static void raid5_free(struct mddev *mddev, void *priv) ++{ ++ struct r5conf *conf = priv; ++ ++ free_conf(conf); ++ mddev->to_remove = &raid5_attrs_group; ++} ++ ++static void status(struct seq_file *seq, struct mddev *mddev) ++{ ++ struct r5conf *conf = mddev->private; ++ int i; ++ ++ seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, ++ mddev->chunk_sectors / 2, mddev->layout); ++ seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); ++ for (i = 0; i < conf->raid_disks; i++) ++ seq_printf (seq, "%s", ++ conf->disks[i].rdev && ++ test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); ++ seq_printf (seq, "]"); ++} ++ ++static void print_raid5_conf (struct r5conf *conf) ++{ ++ int i; ++ struct disk_info *tmp; ++ ++ printk(KERN_DEBUG "RAID conf printout:\n"); ++ if (!conf) { ++ printk("(conf==NULL)\n"); ++ return; ++ } ++ printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, ++ conf->raid_disks, ++ conf->raid_disks - conf->mddev->degraded); ++ ++ for (i = 0; i < conf->raid_disks; i++) { ++ char b[BDEVNAME_SIZE]; ++ tmp = conf->disks + i; ++ if (tmp->rdev) ++ printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n", ++ i, !test_bit(Faulty, &tmp->rdev->flags), ++ bdevname(tmp->rdev->bdev, b)); ++ } ++} ++ ++static int raid5_spare_active(struct mddev *mddev) ++{ ++ int i; ++ struct r5conf *conf = mddev->private; ++ struct disk_info *tmp; ++ int count = 0; ++ unsigned long flags; ++ ++ for (i = 0; i < conf->raid_disks; i++) { ++ tmp = conf->disks + i; ++ if (tmp->replacement ++ && tmp->replacement->recovery_offset == MaxSector ++ && !test_bit(Faulty, &tmp->replacement->flags) ++ && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { ++ /* Replacement has just become active. */ ++ if (!tmp->rdev ++ || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) ++ count++; ++ if (tmp->rdev) { ++ /* Replaced device not technically faulty, ++ * but we need to be sure it gets removed ++ * and never re-added. ++ */ ++ set_bit(Faulty, &tmp->rdev->flags); ++ sysfs_notify_dirent_safe( ++ tmp->rdev->sysfs_state); ++ } ++ sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); ++ } else if (tmp->rdev ++ && tmp->rdev->recovery_offset == MaxSector ++ && !test_bit(Faulty, &tmp->rdev->flags) ++ && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { ++ count++; ++ sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); ++ } ++ } ++ spin_lock_irqsave(&conf->device_lock, flags); ++ mddev->degraded = calc_degraded(conf); ++ spin_unlock_irqrestore(&conf->device_lock, flags); ++ print_raid5_conf(conf); ++ return count; ++} ++ ++static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev) ++{ ++ struct r5conf *conf = mddev->private; ++ int err = 0; ++ int number = rdev->raid_disk; ++ struct md_rdev **rdevp; ++ struct disk_info *p = conf->disks + number; ++ ++ print_raid5_conf(conf); ++ if (rdev == p->rdev) ++ rdevp = &p->rdev; ++ else if (rdev == p->replacement) ++ rdevp = &p->replacement; ++ else ++ return 0; ++ ++ if (number >= conf->raid_disks && ++ conf->reshape_progress == MaxSector) ++ clear_bit(In_sync, &rdev->flags); ++ ++ if (test_bit(In_sync, &rdev->flags) || ++ atomic_read(&rdev->nr_pending)) { ++ err = -EBUSY; ++ goto abort; ++ } ++ /* Only remove non-faulty devices if recovery ++ * isn't possible. ++ */ ++ if (!test_bit(Faulty, &rdev->flags) && ++ mddev->recovery_disabled != conf->recovery_disabled && ++ !has_failed(conf) && ++ (!p->replacement || p->replacement == rdev) && ++ number < conf->raid_disks) { ++ err = -EBUSY; ++ goto abort; ++ } ++ *rdevp = NULL; ++ synchronize_rcu(); ++ if (atomic_read(&rdev->nr_pending)) { ++ /* lost the race, try later */ ++ err = -EBUSY; ++ *rdevp = rdev; ++ } else if (p->replacement) { ++ /* We must have just cleared 'rdev' */ ++ p->rdev = p->replacement; ++ clear_bit(Replacement, &p->replacement->flags); ++ smp_mb(); /* Make sure other CPUs may see both as identical ++ * but will never see neither - if they are careful ++ */ ++ p->replacement = NULL; ++ clear_bit(WantReplacement, &rdev->flags); ++ } else ++ /* We might have just removed the Replacement as faulty- ++ * clear the bit just in case ++ */ ++ clear_bit(WantReplacement, &rdev->flags); ++abort: ++ ++ print_raid5_conf(conf); ++ return err; ++} ++ ++static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) ++{ ++ struct r5conf *conf = mddev->private; ++ int err = -EEXIST; ++ int disk; ++ struct disk_info *p; ++ int first = 0; ++ int last = conf->raid_disks - 1; ++ ++ if (mddev->recovery_disabled == conf->recovery_disabled) ++ return -EBUSY; ++ ++ if (rdev->saved_raid_disk < 0 && has_failed(conf)) ++ /* no point adding a device */ ++ return -EINVAL; ++ ++ if (rdev->raid_disk >= 0) ++ first = last = rdev->raid_disk; ++ ++ /* ++ * find the disk ... but prefer rdev->saved_raid_disk ++ * if possible. ++ */ ++ if (rdev->saved_raid_disk >= 0 && ++ rdev->saved_raid_disk >= first && ++ conf->disks[rdev->saved_raid_disk].rdev == NULL) ++ first = rdev->saved_raid_disk; ++ ++ for (disk = first; disk <= last; disk++) { ++ p = conf->disks + disk; ++ if (p->rdev == NULL) { ++ clear_bit(In_sync, &rdev->flags); ++ rdev->raid_disk = disk; ++ err = 0; ++ if (rdev->saved_raid_disk != disk) ++ conf->fullsync = 1; ++ rcu_assign_pointer(p->rdev, rdev); ++ goto out; ++ } ++ } ++ for (disk = first; disk <= last; disk++) { ++ p = conf->disks + disk; ++ if (test_bit(WantReplacement, &p->rdev->flags) && ++ p->replacement == NULL) { ++ clear_bit(In_sync, &rdev->flags); ++ set_bit(Replacement, &rdev->flags); ++ rdev->raid_disk = disk; ++ err = 0; ++ conf->fullsync = 1; ++ rcu_assign_pointer(p->replacement, rdev); ++ break; ++ } ++ } ++out: ++ print_raid5_conf(conf); ++ return err; ++} ++ ++static int raid5_resize(struct mddev *mddev, sector_t sectors) ++{ ++ /* no resync is happening, and there is enough space ++ * on all devices, so we can resize. ++ * We need to make sure resync covers any new space. ++ * If the array is shrinking we should possibly wait until ++ * any io in the removed space completes, but it hardly seems ++ * worth it. ++ */ ++ sector_t newsize; ++ sectors &= ~((sector_t)mddev->chunk_sectors - 1); ++ newsize = raid5_size(mddev, sectors, mddev->raid_disks); ++ if (mddev->external_size && ++ mddev->array_sectors > newsize) ++ return -EINVAL; ++ if (mddev->bitmap) { ++ int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0); ++ if (ret) ++ return ret; ++ } ++ md_set_array_sectors(mddev, newsize); ++ set_capacity(mddev->gendisk, mddev->array_sectors); ++ revalidate_disk(mddev->gendisk); ++ if (sectors > mddev->dev_sectors && ++ mddev->recovery_cp > mddev->dev_sectors) { ++ mddev->recovery_cp = mddev->dev_sectors; ++ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); ++ } ++ mddev->dev_sectors = sectors; ++ mddev->resync_max_sectors = sectors; ++ return 0; ++} ++ ++static int check_stripe_cache(struct mddev *mddev) ++{ ++ /* Can only proceed if there are plenty of stripe_heads. ++ * We need a minimum of one full stripe,, and for sensible progress ++ * it is best to have about 4 times that. ++ * If we require 4 times, then the default 256 4K stripe_heads will ++ * allow for chunk sizes up to 256K, which is probably OK. ++ * If the chunk size is greater, user-space should request more ++ * stripe_heads first. ++ */ ++ struct r5conf *conf = mddev->private; ++ if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 ++ > conf->min_nr_stripes || ++ ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 ++ > conf->min_nr_stripes) { ++ printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n", ++ mdname(mddev), ++ ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) ++ / STRIPE_SIZE)*4); ++ return 0; ++ } ++ return 1; ++} ++ ++static int check_reshape(struct mddev *mddev) ++{ ++ struct r5conf *conf = mddev->private; ++ ++ if (mddev->delta_disks == 0 && ++ mddev->new_layout == mddev->layout && ++ mddev->new_chunk_sectors == mddev->chunk_sectors) ++ return 0; /* nothing to do */ ++ if (has_failed(conf)) ++ return -EINVAL; ++ if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { ++ /* We might be able to shrink, but the devices must ++ * be made bigger first. ++ * For raid6, 4 is the minimum size. ++ * Otherwise 2 is the minimum ++ */ ++ int min = 2; ++ if (mddev->level == 6) ++ min = 4; ++ if (mddev->raid_disks + mddev->delta_disks < min) ++ return -EINVAL; ++ } ++ ++ if (!check_stripe_cache(mddev)) ++ return -ENOSPC; ++ ++ if (mddev->new_chunk_sectors > mddev->chunk_sectors || ++ mddev->delta_disks > 0) ++ if (resize_chunks(conf, ++ conf->previous_raid_disks ++ + max(0, mddev->delta_disks), ++ max(mddev->new_chunk_sectors, ++ mddev->chunk_sectors) ++ ) < 0) ++ return -ENOMEM; ++ return resize_stripes(conf, (conf->previous_raid_disks ++ + mddev->delta_disks)); ++} ++ ++static int raid5_start_reshape(struct mddev *mddev) ++{ ++ struct r5conf *conf = mddev->private; ++ struct md_rdev *rdev; ++ int spares = 0; ++ unsigned long flags; ++ ++ if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) ++ return -EBUSY; ++ ++ if (!check_stripe_cache(mddev)) ++ return -ENOSPC; ++ ++ if (has_failed(conf)) ++ return -EINVAL; ++ ++ rdev_for_each(rdev, mddev) { ++ if (!test_bit(In_sync, &rdev->flags) ++ && !test_bit(Faulty, &rdev->flags)) ++ spares++; ++ } ++ ++ if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) ++ /* Not enough devices even to make a degraded array ++ * of that size ++ */ ++ return -EINVAL; ++ ++ /* Refuse to reduce size of the array. Any reductions in ++ * array size must be through explicit setting of array_size ++ * attribute. ++ */ ++ if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) ++ < mddev->array_sectors) { ++ printk(KERN_ERR "md/raid:%s: array size must be reduced " ++ "before number of disks\n", mdname(mddev)); ++ return -EINVAL; ++ } ++ ++ atomic_set(&conf->reshape_stripes, 0); ++ spin_lock_irq(&conf->device_lock); ++ write_seqcount_begin(&conf->gen_lock); ++ conf->previous_raid_disks = conf->raid_disks; ++ conf->raid_disks += mddev->delta_disks; ++ conf->prev_chunk_sectors = conf->chunk_sectors; ++ conf->chunk_sectors = mddev->new_chunk_sectors; ++ conf->prev_algo = conf->algorithm; ++ conf->algorithm = mddev->new_layout; ++ conf->generation++; ++ /* Code that selects data_offset needs to see the generation update ++ * if reshape_progress has been set - so a memory barrier needed. ++ */ ++ smp_mb(); ++ if (mddev->reshape_backwards) ++ conf->reshape_progress = raid5_size(mddev, 0, 0); ++ else ++ conf->reshape_progress = 0; ++ conf->reshape_safe = conf->reshape_progress; ++ write_seqcount_end(&conf->gen_lock); ++ spin_unlock_irq(&conf->device_lock); ++ ++ /* Now make sure any requests that proceeded on the assumption ++ * the reshape wasn't running - like Discard or Read - have ++ * completed. ++ */ ++ mddev_suspend(mddev); ++ mddev_resume(mddev); ++ ++ /* Add some new drives, as many as will fit. ++ * We know there are enough to make the newly sized array work. ++ * Don't add devices if we are reducing the number of ++ * devices in the array. This is because it is not possible ++ * to correctly record the "partially reconstructed" state of ++ * such devices during the reshape and confusion could result. ++ */ ++ if (mddev->delta_disks >= 0) { ++ rdev_for_each(rdev, mddev) ++ if (rdev->raid_disk < 0 && ++ !test_bit(Faulty, &rdev->flags)) { ++ if (raid5_add_disk(mddev, rdev) == 0) { ++ if (rdev->raid_disk ++ >= conf->previous_raid_disks) ++ set_bit(In_sync, &rdev->flags); ++ else ++ rdev->recovery_offset = 0; ++ ++ if (sysfs_link_rdev(mddev, rdev)) ++ /* Failure here is OK */; ++ } ++ } else if (rdev->raid_disk >= conf->previous_raid_disks ++ && !test_bit(Faulty, &rdev->flags)) { ++ /* This is a spare that was manually added */ ++ set_bit(In_sync, &rdev->flags); ++ } ++ ++ /* When a reshape changes the number of devices, ++ * ->degraded is measured against the larger of the ++ * pre and post number of devices. ++ */ ++ spin_lock_irqsave(&conf->device_lock, flags); ++ mddev->degraded = calc_degraded(conf); ++ spin_unlock_irqrestore(&conf->device_lock, flags); ++ } ++ mddev->raid_disks = conf->raid_disks; ++ mddev->reshape_position = conf->reshape_progress; ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ ++ clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); ++ clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); ++ clear_bit(MD_RECOVERY_DONE, &mddev->recovery); ++ set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); ++ set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); ++ mddev->sync_thread = md_register_thread(md_do_sync, mddev, ++ "reshape"); ++ if (!mddev->sync_thread) { ++ mddev->recovery = 0; ++ spin_lock_irq(&conf->device_lock); ++ write_seqcount_begin(&conf->gen_lock); ++ mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; ++ mddev->new_chunk_sectors = ++ conf->chunk_sectors = conf->prev_chunk_sectors; ++ mddev->new_layout = conf->algorithm = conf->prev_algo; ++ rdev_for_each(rdev, mddev) ++ rdev->new_data_offset = rdev->data_offset; ++ smp_wmb(); ++ conf->generation --; ++ conf->reshape_progress = MaxSector; ++ mddev->reshape_position = MaxSector; ++ write_seqcount_end(&conf->gen_lock); ++ spin_unlock_irq(&conf->device_lock); ++ return -EAGAIN; ++ } ++ conf->reshape_checkpoint = jiffies; ++ md_wakeup_thread(mddev->sync_thread); ++ md_new_event(mddev); ++ return 0; ++} ++ ++/* This is called from the reshape thread and should make any ++ * changes needed in 'conf' ++ */ ++static void end_reshape(struct r5conf *conf) ++{ ++ ++ if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { ++ struct md_rdev *rdev; ++ ++ spin_lock_irq(&conf->device_lock); ++ conf->previous_raid_disks = conf->raid_disks; ++ rdev_for_each(rdev, conf->mddev) ++ rdev->data_offset = rdev->new_data_offset; ++ smp_wmb(); ++ conf->reshape_progress = MaxSector; ++ spin_unlock_irq(&conf->device_lock); ++ wake_up(&conf->wait_for_overlap); ++ ++ /* read-ahead size must cover two whole stripes, which is ++ * 2 * (datadisks) * chunksize where 'n' is the number of raid devices ++ */ ++ if (conf->mddev->queue) { ++ int data_disks = conf->raid_disks - conf->max_degraded; ++ int stripe = data_disks * ((conf->chunk_sectors << 9) ++ / PAGE_SIZE); ++ if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) ++ conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; ++ } ++ } ++} ++ ++/* This is called from the raid5d thread with mddev_lock held. ++ * It makes config changes to the device. ++ */ ++static void raid5_finish_reshape(struct mddev *mddev) ++{ ++ struct r5conf *conf = mddev->private; ++ ++ if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { ++ ++ if (mddev->delta_disks > 0) { ++ md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); ++ set_capacity(mddev->gendisk, mddev->array_sectors); ++ revalidate_disk(mddev->gendisk); ++ } else { ++ int d; ++ spin_lock_irq(&conf->device_lock); ++ mddev->degraded = calc_degraded(conf); ++ spin_unlock_irq(&conf->device_lock); ++ for (d = conf->raid_disks ; ++ d < conf->raid_disks - mddev->delta_disks; ++ d++) { ++ struct md_rdev *rdev = conf->disks[d].rdev; ++ if (rdev) ++ clear_bit(In_sync, &rdev->flags); ++ rdev = conf->disks[d].replacement; ++ if (rdev) ++ clear_bit(In_sync, &rdev->flags); ++ } ++ } ++ mddev->layout = conf->algorithm; ++ mddev->chunk_sectors = conf->chunk_sectors; ++ mddev->reshape_position = MaxSector; ++ mddev->delta_disks = 0; ++ mddev->reshape_backwards = 0; ++ } ++} ++ ++static void raid5_quiesce(struct mddev *mddev, int state) ++{ ++ struct r5conf *conf = mddev->private; ++ ++ switch(state) { ++ case 2: /* resume for a suspend */ ++ wake_up(&conf->wait_for_overlap); ++ break; ++ ++ case 1: /* stop all writes */ ++ lock_all_device_hash_locks_irq(conf); ++ /* '2' tells resync/reshape to pause so that all ++ * active stripes can drain ++ */ ++ conf->quiesce = 2; ++ wait_event_cmd(conf->wait_for_stripe, ++ atomic_read(&conf->active_stripes) == 0 && ++ atomic_read(&conf->active_aligned_reads) == 0, ++ unlock_all_device_hash_locks_irq(conf), ++ lock_all_device_hash_locks_irq(conf)); ++ conf->quiesce = 1; ++ unlock_all_device_hash_locks_irq(conf); ++ /* allow reshape to continue */ ++ wake_up(&conf->wait_for_overlap); ++ break; ++ ++ case 0: /* re-enable writes */ ++ lock_all_device_hash_locks_irq(conf); ++ conf->quiesce = 0; ++ wake_up(&conf->wait_for_stripe); ++ wake_up(&conf->wait_for_overlap); ++ unlock_all_device_hash_locks_irq(conf); ++ break; ++ } ++} ++ ++static void *raid45_takeover_raid0(struct mddev *mddev, int level) ++{ ++ struct r0conf *raid0_conf = mddev->private; ++ sector_t sectors; ++ ++ /* for raid0 takeover only one zone is supported */ ++ if (raid0_conf->nr_strip_zones > 1) { ++ printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n", ++ mdname(mddev)); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ sectors = raid0_conf->strip_zone[0].zone_end; ++ sector_div(sectors, raid0_conf->strip_zone[0].nb_dev); ++ mddev->dev_sectors = sectors; ++ mddev->new_level = level; ++ mddev->new_layout = ALGORITHM_PARITY_N; ++ mddev->new_chunk_sectors = mddev->chunk_sectors; ++ mddev->raid_disks += 1; ++ mddev->delta_disks = 1; ++ /* make sure it will be not marked as dirty */ ++ mddev->recovery_cp = MaxSector; ++ ++ return setup_conf(mddev); ++} ++ ++static void *raid5_takeover_raid1(struct mddev *mddev) ++{ ++ int chunksect; ++ ++ if (mddev->raid_disks != 2 || ++ mddev->degraded > 1) ++ return ERR_PTR(-EINVAL); ++ ++ /* Should check if there are write-behind devices? */ ++ ++ chunksect = 64*2; /* 64K by default */ ++ ++ /* The array must be an exact multiple of chunksize */ ++ while (chunksect && (mddev->array_sectors & (chunksect-1))) ++ chunksect >>= 1; ++ ++ if ((chunksect<<9) < STRIPE_SIZE) ++ /* array size does not allow a suitable chunk size */ ++ return ERR_PTR(-EINVAL); ++ ++ mddev->new_level = 5; ++ mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; ++ mddev->new_chunk_sectors = chunksect; ++ ++ return setup_conf(mddev); ++} ++ ++static void *raid5_takeover_raid6(struct mddev *mddev) ++{ ++ int new_layout; ++ ++ switch (mddev->layout) { ++ case ALGORITHM_LEFT_ASYMMETRIC_6: ++ new_layout = ALGORITHM_LEFT_ASYMMETRIC; ++ break; ++ case ALGORITHM_RIGHT_ASYMMETRIC_6: ++ new_layout = ALGORITHM_RIGHT_ASYMMETRIC; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC_6: ++ new_layout = ALGORITHM_LEFT_SYMMETRIC; ++ break; ++ case ALGORITHM_RIGHT_SYMMETRIC_6: ++ new_layout = ALGORITHM_RIGHT_SYMMETRIC; ++ break; ++ case ALGORITHM_PARITY_0_6: ++ new_layout = ALGORITHM_PARITY_0; ++ break; ++ case ALGORITHM_PARITY_N: ++ new_layout = ALGORITHM_PARITY_N; ++ break; ++ default: ++ return ERR_PTR(-EINVAL); ++ } ++ mddev->new_level = 5; ++ mddev->new_layout = new_layout; ++ mddev->delta_disks = -1; ++ mddev->raid_disks -= 1; ++ return setup_conf(mddev); ++} ++ ++static int raid5_check_reshape(struct mddev *mddev) ++{ ++ /* For a 2-drive array, the layout and chunk size can be changed ++ * immediately as not restriping is needed. ++ * For larger arrays we record the new value - after validation ++ * to be used by a reshape pass. ++ */ ++ struct r5conf *conf = mddev->private; ++ int new_chunk = mddev->new_chunk_sectors; ++ ++ if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) ++ return -EINVAL; ++ if (new_chunk > 0) { ++ if (!is_power_of_2(new_chunk)) ++ return -EINVAL; ++ if (new_chunk < (PAGE_SIZE>>9)) ++ return -EINVAL; ++ if (mddev->array_sectors & (new_chunk-1)) ++ /* not factor of array size */ ++ return -EINVAL; ++ } ++ ++ /* They look valid */ ++ ++ if (mddev->raid_disks == 2) { ++ /* can make the change immediately */ ++ if (mddev->new_layout >= 0) { ++ conf->algorithm = mddev->new_layout; ++ mddev->layout = mddev->new_layout; ++ } ++ if (new_chunk > 0) { ++ conf->chunk_sectors = new_chunk ; ++ mddev->chunk_sectors = new_chunk; ++ } ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ md_wakeup_thread(mddev->thread); ++ } ++ return check_reshape(mddev); ++} ++ ++static int raid6_check_reshape(struct mddev *mddev) ++{ ++ int new_chunk = mddev->new_chunk_sectors; ++ ++ if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) ++ return -EINVAL; ++ if (new_chunk > 0) { ++ if (!is_power_of_2(new_chunk)) ++ return -EINVAL; ++ if (new_chunk < (PAGE_SIZE >> 9)) ++ return -EINVAL; ++ if (mddev->array_sectors & (new_chunk-1)) ++ /* not factor of array size */ ++ return -EINVAL; ++ } ++ ++ /* They look valid */ ++ return check_reshape(mddev); ++} ++ ++static void *raid5_takeover(struct mddev *mddev) ++{ ++ /* raid5 can take over: ++ * raid0 - if there is only one strip zone - make it a raid4 layout ++ * raid1 - if there are two drives. We need to know the chunk size ++ * raid4 - trivial - just use a raid4 layout. ++ * raid6 - Providing it is a *_6 layout ++ */ ++ if (mddev->level == 0) ++ return raid45_takeover_raid0(mddev, 5); ++ if (mddev->level == 1) ++ return raid5_takeover_raid1(mddev); ++ if (mddev->level == 4) { ++ mddev->new_layout = ALGORITHM_PARITY_N; ++ mddev->new_level = 5; ++ return setup_conf(mddev); ++ } ++ if (mddev->level == 6) ++ return raid5_takeover_raid6(mddev); ++ ++ return ERR_PTR(-EINVAL); ++} ++ ++static void *raid4_takeover(struct mddev *mddev) ++{ ++ /* raid4 can take over: ++ * raid0 - if there is only one strip zone ++ * raid5 - if layout is right ++ */ ++ if (mddev->level == 0) ++ return raid45_takeover_raid0(mddev, 4); ++ if (mddev->level == 5 && ++ mddev->layout == ALGORITHM_PARITY_N) { ++ mddev->new_layout = 0; ++ mddev->new_level = 4; ++ return setup_conf(mddev); ++ } ++ return ERR_PTR(-EINVAL); ++} ++ ++static struct md_personality raid5_personality; ++ ++static void *raid6_takeover(struct mddev *mddev) ++{ ++ /* Currently can only take over a raid5. We map the ++ * personality to an equivalent raid6 personality ++ * with the Q block at the end. ++ */ ++ int new_layout; ++ ++ if (mddev->pers != &raid5_personality) ++ return ERR_PTR(-EINVAL); ++ if (mddev->degraded > 1) ++ return ERR_PTR(-EINVAL); ++ if (mddev->raid_disks > 253) ++ return ERR_PTR(-EINVAL); ++ if (mddev->raid_disks < 3) ++ return ERR_PTR(-EINVAL); ++ ++ switch (mddev->layout) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; ++ break; ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ new_layout = ALGORITHM_LEFT_SYMMETRIC_6; ++ break; ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; ++ break; ++ case ALGORITHM_PARITY_0: ++ new_layout = ALGORITHM_PARITY_0_6; ++ break; ++ case ALGORITHM_PARITY_N: ++ new_layout = ALGORITHM_PARITY_N; ++ break; ++ default: ++ return ERR_PTR(-EINVAL); ++ } ++ mddev->new_level = 6; ++ mddev->new_layout = new_layout; ++ mddev->delta_disks = 1; ++ mddev->raid_disks += 1; ++ return setup_conf(mddev); ++} ++ ++static struct md_personality raid6_personality = ++{ ++ .name = "raid6", ++ .level = 6, ++ .owner = THIS_MODULE, ++ .make_request = make_request, ++ .run = run, ++ .free = raid5_free, ++ .status = status, ++ .error_handler = error, ++ .hot_add_disk = raid5_add_disk, ++ .hot_remove_disk= raid5_remove_disk, ++ .spare_active = raid5_spare_active, ++ .sync_request = sync_request, ++ .resize = raid5_resize, ++ .size = raid5_size, ++ .check_reshape = raid6_check_reshape, ++ .start_reshape = raid5_start_reshape, ++ .finish_reshape = raid5_finish_reshape, ++ .quiesce = raid5_quiesce, ++ .takeover = raid6_takeover, ++ .congested = raid5_congested, ++ .mergeable_bvec = raid5_mergeable_bvec, ++}; ++static struct md_personality raid5_personality = ++{ ++ .name = "raid5", ++ .level = 5, ++ .owner = THIS_MODULE, ++ .make_request = make_request, ++ .run = run, ++ .free = raid5_free, ++ .status = status, ++ .error_handler = error, ++ .hot_add_disk = raid5_add_disk, ++ .hot_remove_disk= raid5_remove_disk, ++ .spare_active = raid5_spare_active, ++ .sync_request = sync_request, ++ .resize = raid5_resize, ++ .size = raid5_size, ++ .check_reshape = raid5_check_reshape, ++ .start_reshape = raid5_start_reshape, ++ .finish_reshape = raid5_finish_reshape, ++ .quiesce = raid5_quiesce, ++ .takeover = raid5_takeover, ++ .congested = raid5_congested, ++ .mergeable_bvec = raid5_mergeable_bvec, ++}; ++ ++static struct md_personality raid4_personality = ++{ ++ .name = "raid4", ++ .level = 4, ++ .owner = THIS_MODULE, ++ .make_request = make_request, ++ .run = run, ++ .free = raid5_free, ++ .status = status, ++ .error_handler = error, ++ .hot_add_disk = raid5_add_disk, ++ .hot_remove_disk= raid5_remove_disk, ++ .spare_active = raid5_spare_active, ++ .sync_request = sync_request, ++ .resize = raid5_resize, ++ .size = raid5_size, ++ .check_reshape = raid5_check_reshape, ++ .start_reshape = raid5_start_reshape, ++ .finish_reshape = raid5_finish_reshape, ++ .quiesce = raid5_quiesce, ++ .takeover = raid4_takeover, ++ .congested = raid5_congested, ++ .mergeable_bvec = raid5_mergeable_bvec, ++}; ++ ++static int __init raid5_init(void) ++{ ++ raid5_wq = alloc_workqueue("raid5wq", ++ WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0); ++ if (!raid5_wq) ++ return -ENOMEM; ++ register_md_personality(&raid6_personality); ++ register_md_personality(&raid5_personality); ++ register_md_personality(&raid4_personality); ++ return 0; ++} ++ ++static void raid5_exit(void) ++{ ++ unregister_md_personality(&raid6_personality); ++ unregister_md_personality(&raid5_personality); ++ unregister_md_personality(&raid4_personality); ++ destroy_workqueue(raid5_wq); ++} ++ ++module_init(raid5_init); ++module_exit(raid5_exit); ++MODULE_LICENSE("GPL"); ++MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); ++MODULE_ALIAS("md-personality-4"); /* RAID5 */ ++MODULE_ALIAS("md-raid5"); ++MODULE_ALIAS("md-raid4"); ++MODULE_ALIAS("md-level-5"); ++MODULE_ALIAS("md-level-4"); ++MODULE_ALIAS("md-personality-8"); /* RAID6 */ ++MODULE_ALIAS("md-raid6"); ++MODULE_ALIAS("md-level-6"); ++ ++/* This used to be two separate modules, they were: */ ++MODULE_ALIAS("raid5"); ++MODULE_ALIAS("raid6"); +diff -Nur linux-4.1.10.orig/drivers/md/raid5.h linux-4.1.10/drivers/md/raid5.h +--- linux-4.1.10.orig/drivers/md/raid5.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/md/raid5.h 2015-10-07 18:00:08.000000000 +0200 +@@ -495,6 +495,7 @@ int recovery_disabled; /* per cpu variables */ struct raid5_percpu { @@ -6407,9 +14286,621 @@ diff -Nur linux-4.1.6.orig/drivers/md/raid5.h linux-4.1.6/drivers/md/raid5.h struct page *spare_page; /* Used when checking P/Q in raid6 */ struct flex_array *scribble; /* space for constructing buffer * lists and performing address -diff -Nur linux-4.1.6.orig/drivers/misc/hwlat_detector.c linux-4.1.6/drivers/misc/hwlat_detector.c ---- linux-4.1.6.orig/drivers/misc/hwlat_detector.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/drivers/misc/hwlat_detector.c 2015-09-08 23:49:06.082091803 +0200 +diff -Nur linux-4.1.10.orig/drivers/md/raid5.h.orig linux-4.1.10/drivers/md/raid5.h.orig +--- linux-4.1.10.orig/drivers/md/raid5.h.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/drivers/md/raid5.h.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,608 @@ ++#ifndef _RAID5_H ++#define _RAID5_H ++ ++#include <linux/raid/xor.h> ++#include <linux/dmaengine.h> ++ ++/* ++ * ++ * Each stripe contains one buffer per device. Each buffer can be in ++ * one of a number of states stored in "flags". Changes between ++ * these states happen *almost* exclusively under the protection of the ++ * STRIPE_ACTIVE flag. Some very specific changes can happen in bi_end_io, and ++ * these are not protected by STRIPE_ACTIVE. ++ * ++ * The flag bits that are used to represent these states are: ++ * R5_UPTODATE and R5_LOCKED ++ * ++ * State Empty == !UPTODATE, !LOCK ++ * We have no data, and there is no active request ++ * State Want == !UPTODATE, LOCK ++ * A read request is being submitted for this block ++ * State Dirty == UPTODATE, LOCK ++ * Some new data is in this buffer, and it is being written out ++ * State Clean == UPTODATE, !LOCK ++ * We have valid data which is the same as on disc ++ * ++ * The possible state transitions are: ++ * ++ * Empty -> Want - on read or write to get old data for parity calc ++ * Empty -> Dirty - on compute_parity to satisfy write/sync request. ++ * Empty -> Clean - on compute_block when computing a block for failed drive ++ * Want -> Empty - on failed read ++ * Want -> Clean - on successful completion of read request ++ * Dirty -> Clean - on successful completion of write request ++ * Dirty -> Clean - on failed write ++ * Clean -> Dirty - on compute_parity to satisfy write/sync (RECONSTRUCT or RMW) ++ * ++ * The Want->Empty, Want->Clean, Dirty->Clean, transitions ++ * all happen in b_end_io at interrupt time. ++ * Each sets the Uptodate bit before releasing the Lock bit. ++ * This leaves one multi-stage transition: ++ * Want->Dirty->Clean ++ * This is safe because thinking that a Clean buffer is actually dirty ++ * will at worst delay some action, and the stripe will be scheduled ++ * for attention after the transition is complete. ++ * ++ * There is one possibility that is not covered by these states. That ++ * is if one drive has failed and there is a spare being rebuilt. We ++ * can't distinguish between a clean block that has been generated ++ * from parity calculations, and a clean block that has been ++ * successfully written to the spare ( or to parity when resyncing). ++ * To distinguish these states we have a stripe bit STRIPE_INSYNC that ++ * is set whenever a write is scheduled to the spare, or to the parity ++ * disc if there is no spare. A sync request clears this bit, and ++ * when we find it set with no buffers locked, we know the sync is ++ * complete. ++ * ++ * Buffers for the md device that arrive via make_request are attached ++ * to the appropriate stripe in one of two lists linked on b_reqnext. ++ * One list (bh_read) for read requests, one (bh_write) for write. ++ * There should never be more than one buffer on the two lists ++ * together, but we are not guaranteed of that so we allow for more. ++ * ++ * If a buffer is on the read list when the associated cache buffer is ++ * Uptodate, the data is copied into the read buffer and it's b_end_io ++ * routine is called. This may happen in the end_request routine only ++ * if the buffer has just successfully been read. end_request should ++ * remove the buffers from the list and then set the Uptodate bit on ++ * the buffer. Other threads may do this only if they first check ++ * that the Uptodate bit is set. Once they have checked that they may ++ * take buffers off the read queue. ++ * ++ * When a buffer on the write list is committed for write it is copied ++ * into the cache buffer, which is then marked dirty, and moved onto a ++ * third list, the written list (bh_written). Once both the parity ++ * block and the cached buffer are successfully written, any buffer on ++ * a written list can be returned with b_end_io. ++ * ++ * The write list and read list both act as fifos. The read list, ++ * write list and written list are protected by the device_lock. ++ * The device_lock is only for list manipulations and will only be ++ * held for a very short time. It can be claimed from interrupts. ++ * ++ * ++ * Stripes in the stripe cache can be on one of two lists (or on ++ * neither). The "inactive_list" contains stripes which are not ++ * currently being used for any request. They can freely be reused ++ * for another stripe. The "handle_list" contains stripes that need ++ * to be handled in some way. Both of these are fifo queues. Each ++ * stripe is also (potentially) linked to a hash bucket in the hash ++ * table so that it can be found by sector number. Stripes that are ++ * not hashed must be on the inactive_list, and will normally be at ++ * the front. All stripes start life this way. ++ * ++ * The inactive_list, handle_list and hash bucket lists are all protected by the ++ * device_lock. ++ * - stripes have a reference counter. If count==0, they are on a list. ++ * - If a stripe might need handling, STRIPE_HANDLE is set. ++ * - When refcount reaches zero, then if STRIPE_HANDLE it is put on ++ * handle_list else inactive_list ++ * ++ * This, combined with the fact that STRIPE_HANDLE is only ever ++ * cleared while a stripe has a non-zero count means that if the ++ * refcount is 0 and STRIPE_HANDLE is set, then it is on the ++ * handle_list and if recount is 0 and STRIPE_HANDLE is not set, then ++ * the stripe is on inactive_list. ++ * ++ * The possible transitions are: ++ * activate an unhashed/inactive stripe (get_active_stripe()) ++ * lockdev check-hash unlink-stripe cnt++ clean-stripe hash-stripe unlockdev ++ * activate a hashed, possibly active stripe (get_active_stripe()) ++ * lockdev check-hash if(!cnt++)unlink-stripe unlockdev ++ * attach a request to an active stripe (add_stripe_bh()) ++ * lockdev attach-buffer unlockdev ++ * handle a stripe (handle_stripe()) ++ * setSTRIPE_ACTIVE, clrSTRIPE_HANDLE ... ++ * (lockdev check-buffers unlockdev) .. ++ * change-state .. ++ * record io/ops needed clearSTRIPE_ACTIVE schedule io/ops ++ * release an active stripe (release_stripe()) ++ * lockdev if (!--cnt) { if STRIPE_HANDLE, add to handle_list else add to inactive-list } unlockdev ++ * ++ * The refcount counts each thread that have activated the stripe, ++ * plus raid5d if it is handling it, plus one for each active request ++ * on a cached buffer, and plus one if the stripe is undergoing stripe ++ * operations. ++ * ++ * The stripe operations are: ++ * -copying data between the stripe cache and user application buffers ++ * -computing blocks to save a disk access, or to recover a missing block ++ * -updating the parity on a write operation (reconstruct write and ++ * read-modify-write) ++ * -checking parity correctness ++ * -running i/o to disk ++ * These operations are carried out by raid5_run_ops which uses the async_tx ++ * api to (optionally) offload operations to dedicated hardware engines. ++ * When requesting an operation handle_stripe sets the pending bit for the ++ * operation and increments the count. raid5_run_ops is then run whenever ++ * the count is non-zero. ++ * There are some critical dependencies between the operations that prevent some ++ * from being requested while another is in flight. ++ * 1/ Parity check operations destroy the in cache version of the parity block, ++ * so we prevent parity dependent operations like writes and compute_blocks ++ * from starting while a check is in progress. Some dma engines can perform ++ * the check without damaging the parity block, in these cases the parity ++ * block is re-marked up to date (assuming the check was successful) and is ++ * not re-read from disk. ++ * 2/ When a write operation is requested we immediately lock the affected ++ * blocks, and mark them as not up to date. This causes new read requests ++ * to be held off, as well as parity checks and compute block operations. ++ * 3/ Once a compute block operation has been requested handle_stripe treats ++ * that block as if it is up to date. raid5_run_ops guaruntees that any ++ * operation that is dependent on the compute block result is initiated after ++ * the compute block completes. ++ */ ++ ++/* ++ * Operations state - intermediate states that are visible outside of ++ * STRIPE_ACTIVE. ++ * In general _idle indicates nothing is running, _run indicates a data ++ * processing operation is active, and _result means the data processing result ++ * is stable and can be acted upon. For simple operations like biofill and ++ * compute that only have an _idle and _run state they are indicated with ++ * sh->state flags (STRIPE_BIOFILL_RUN and STRIPE_COMPUTE_RUN) ++ */ ++/** ++ * enum check_states - handles syncing / repairing a stripe ++ * @check_state_idle - check operations are quiesced ++ * @check_state_run - check operation is running ++ * @check_state_result - set outside lock when check result is valid ++ * @check_state_compute_run - check failed and we are repairing ++ * @check_state_compute_result - set outside lock when compute result is valid ++ */ ++enum check_states { ++ check_state_idle = 0, ++ check_state_run, /* xor parity check */ ++ check_state_run_q, /* q-parity check */ ++ check_state_run_pq, /* pq dual parity check */ ++ check_state_check_result, ++ check_state_compute_run, /* parity repair */ ++ check_state_compute_result, ++}; ++ ++/** ++ * enum reconstruct_states - handles writing or expanding a stripe ++ */ ++enum reconstruct_states { ++ reconstruct_state_idle = 0, ++ reconstruct_state_prexor_drain_run, /* prexor-write */ ++ reconstruct_state_drain_run, /* write */ ++ reconstruct_state_run, /* expand */ ++ reconstruct_state_prexor_drain_result, ++ reconstruct_state_drain_result, ++ reconstruct_state_result, ++}; ++ ++struct stripe_head { ++ struct hlist_node hash; ++ struct list_head lru; /* inactive_list or handle_list */ ++ struct llist_node release_list; ++ struct r5conf *raid_conf; ++ short generation; /* increments with every ++ * reshape */ ++ sector_t sector; /* sector of this row */ ++ short pd_idx; /* parity disk index */ ++ short qd_idx; /* 'Q' disk index for raid6 */ ++ short ddf_layout;/* use DDF ordering to calculate Q */ ++ short hash_lock_index; ++ unsigned long state; /* state flags */ ++ atomic_t count; /* nr of active thread/requests */ ++ int bm_seq; /* sequence number for bitmap flushes */ ++ int disks; /* disks in stripe */ ++ int overwrite_disks; /* total overwrite disks in stripe, ++ * this is only checked when stripe ++ * has STRIPE_BATCH_READY ++ */ ++ enum check_states check_state; ++ enum reconstruct_states reconstruct_state; ++ spinlock_t stripe_lock; ++ int cpu; ++ struct r5worker_group *group; ++ ++ struct stripe_head *batch_head; /* protected by stripe lock */ ++ spinlock_t batch_lock; /* only header's lock is useful */ ++ struct list_head batch_list; /* protected by head's batch lock*/ ++ /** ++ * struct stripe_operations ++ * @target - STRIPE_OP_COMPUTE_BLK target ++ * @target2 - 2nd compute target in the raid6 case ++ * @zero_sum_result - P and Q verification flags ++ * @request - async service request flags for raid_run_ops ++ */ ++ struct stripe_operations { ++ int target, target2; ++ enum sum_check_flags zero_sum_result; ++ } ops; ++ struct r5dev { ++ /* rreq and rvec are used for the replacement device when ++ * writing data to both devices. ++ */ ++ struct bio req, rreq; ++ struct bio_vec vec, rvec; ++ struct page *page, *orig_page; ++ struct bio *toread, *read, *towrite, *written; ++ sector_t sector; /* sector of this page */ ++ unsigned long flags; ++ } dev[1]; /* allocated with extra space depending of RAID geometry */ ++}; ++ ++/* stripe_head_state - collects and tracks the dynamic state of a stripe_head ++ * for handle_stripe. ++ */ ++struct stripe_head_state { ++ /* 'syncing' means that we need to read all devices, either ++ * to check/correct parity, or to reconstruct a missing device. ++ * 'replacing' means we are replacing one or more drives and ++ * the source is valid at this point so we don't need to ++ * read all devices, just the replacement targets. ++ */ ++ int syncing, expanding, expanded, replacing; ++ int locked, uptodate, to_read, to_write, failed, written; ++ int to_fill, compute, req_compute, non_overwrite; ++ int failed_num[2]; ++ int p_failed, q_failed; ++ int dec_preread_active; ++ unsigned long ops_request; ++ ++ struct bio *return_bi; ++ struct md_rdev *blocked_rdev; ++ int handle_bad_blocks; ++}; ++ ++/* Flags for struct r5dev.flags */ ++enum r5dev_flags { ++ R5_UPTODATE, /* page contains current data */ ++ R5_LOCKED, /* IO has been submitted on "req" */ ++ R5_DOUBLE_LOCKED,/* Cannot clear R5_LOCKED until 2 writes complete */ ++ R5_OVERWRITE, /* towrite covers whole page */ ++/* and some that are internal to handle_stripe */ ++ R5_Insync, /* rdev && rdev->in_sync at start */ ++ R5_Wantread, /* want to schedule a read */ ++ R5_Wantwrite, ++ R5_Overlap, /* There is a pending overlapping request ++ * on this block */ ++ R5_ReadNoMerge, /* prevent bio from merging in block-layer */ ++ R5_ReadError, /* seen a read error here recently */ ++ R5_ReWrite, /* have tried to over-write the readerror */ ++ ++ R5_Expanded, /* This block now has post-expand data */ ++ R5_Wantcompute, /* compute_block in progress treat as ++ * uptodate ++ */ ++ R5_Wantfill, /* dev->toread contains a bio that needs ++ * filling ++ */ ++ R5_Wantdrain, /* dev->towrite needs to be drained */ ++ R5_WantFUA, /* Write should be FUA */ ++ R5_SyncIO, /* The IO is sync */ ++ R5_WriteError, /* got a write error - need to record it */ ++ R5_MadeGood, /* A bad block has been fixed by writing to it */ ++ R5_ReadRepl, /* Will/did read from replacement rather than orig */ ++ R5_MadeGoodRepl,/* A bad block on the replacement device has been ++ * fixed by writing to it */ ++ R5_NeedReplace, /* This device has a replacement which is not ++ * up-to-date at this stripe. */ ++ R5_WantReplace, /* We need to update the replacement, we have read ++ * data in, and now is a good time to write it out. ++ */ ++ R5_Discard, /* Discard the stripe */ ++ R5_SkipCopy, /* Don't copy data from bio to stripe cache */ ++}; ++ ++/* ++ * Stripe state ++ */ ++enum { ++ STRIPE_ACTIVE, ++ STRIPE_HANDLE, ++ STRIPE_SYNC_REQUESTED, ++ STRIPE_SYNCING, ++ STRIPE_INSYNC, ++ STRIPE_REPLACED, ++ STRIPE_PREREAD_ACTIVE, ++ STRIPE_DELAYED, ++ STRIPE_DEGRADED, ++ STRIPE_BIT_DELAY, ++ STRIPE_EXPANDING, ++ STRIPE_EXPAND_SOURCE, ++ STRIPE_EXPAND_READY, ++ STRIPE_IO_STARTED, /* do not count towards 'bypass_count' */ ++ STRIPE_FULL_WRITE, /* all blocks are set to be overwritten */ ++ STRIPE_BIOFILL_RUN, ++ STRIPE_COMPUTE_RUN, ++ STRIPE_OPS_REQ_PENDING, ++ STRIPE_ON_UNPLUG_LIST, ++ STRIPE_DISCARD, ++ STRIPE_ON_RELEASE_LIST, ++ STRIPE_BATCH_READY, ++ STRIPE_BATCH_ERR, ++ STRIPE_BITMAP_PENDING, /* Being added to bitmap, don't add ++ * to batch yet. ++ */ ++}; ++ ++#define STRIPE_EXPAND_SYNC_FLAGS \ ++ ((1 << STRIPE_EXPAND_SOURCE) |\ ++ (1 << STRIPE_EXPAND_READY) |\ ++ (1 << STRIPE_EXPANDING) |\ ++ (1 << STRIPE_SYNC_REQUESTED)) ++/* ++ * Operation request flags ++ */ ++enum { ++ STRIPE_OP_BIOFILL, ++ STRIPE_OP_COMPUTE_BLK, ++ STRIPE_OP_PREXOR, ++ STRIPE_OP_BIODRAIN, ++ STRIPE_OP_RECONSTRUCT, ++ STRIPE_OP_CHECK, ++}; ++ ++/* ++ * RAID parity calculation preferences ++ */ ++enum { ++ PARITY_DISABLE_RMW = 0, ++ PARITY_ENABLE_RMW, ++ PARITY_PREFER_RMW, ++}; ++ ++/* ++ * Pages requested from set_syndrome_sources() ++ */ ++enum { ++ SYNDROME_SRC_ALL, ++ SYNDROME_SRC_WANT_DRAIN, ++ SYNDROME_SRC_WRITTEN, ++}; ++/* ++ * Plugging: ++ * ++ * To improve write throughput, we need to delay the handling of some ++ * stripes until there has been a chance that several write requests ++ * for the one stripe have all been collected. ++ * In particular, any write request that would require pre-reading ++ * is put on a "delayed" queue until there are no stripes currently ++ * in a pre-read phase. Further, if the "delayed" queue is empty when ++ * a stripe is put on it then we "plug" the queue and do not process it ++ * until an unplug call is made. (the unplug_io_fn() is called). ++ * ++ * When preread is initiated on a stripe, we set PREREAD_ACTIVE and add ++ * it to the count of prereading stripes. ++ * When write is initiated, or the stripe refcnt == 0 (just in case) we ++ * clear the PREREAD_ACTIVE flag and decrement the count ++ * Whenever the 'handle' queue is empty and the device is not plugged, we ++ * move any strips from delayed to handle and clear the DELAYED flag and set ++ * PREREAD_ACTIVE. ++ * In stripe_handle, if we find pre-reading is necessary, we do it if ++ * PREREAD_ACTIVE is set, else we set DELAYED which will send it to the delayed queue. ++ * HANDLE gets cleared if stripe_handle leaves nothing locked. ++ */ ++ ++struct disk_info { ++ struct md_rdev *rdev, *replacement; ++}; ++ ++/* NOTE NR_STRIPE_HASH_LOCKS must remain below 64. ++ * This is because we sometimes take all the spinlocks ++ * and creating that much locking depth can cause ++ * problems. ++ */ ++#define NR_STRIPE_HASH_LOCKS 8 ++#define STRIPE_HASH_LOCKS_MASK (NR_STRIPE_HASH_LOCKS - 1) ++ ++struct r5worker { ++ struct work_struct work; ++ struct r5worker_group *group; ++ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; ++ bool working; ++}; ++ ++struct r5worker_group { ++ struct list_head handle_list; ++ struct r5conf *conf; ++ struct r5worker *workers; ++ int stripes_cnt; ++}; ++ ++struct r5conf { ++ struct hlist_head *stripe_hashtbl; ++ /* only protect corresponding hash list and inactive_list */ ++ spinlock_t hash_locks[NR_STRIPE_HASH_LOCKS]; ++ struct mddev *mddev; ++ int chunk_sectors; ++ int level, algorithm, rmw_level; ++ int max_degraded; ++ int raid_disks; ++ int max_nr_stripes; ++ int min_nr_stripes; ++ ++ /* reshape_progress is the leading edge of a 'reshape' ++ * It has value MaxSector when no reshape is happening ++ * If delta_disks < 0, it is the last sector we started work on, ++ * else is it the next sector to work on. ++ */ ++ sector_t reshape_progress; ++ /* reshape_safe is the trailing edge of a reshape. We know that ++ * before (or after) this address, all reshape has completed. ++ */ ++ sector_t reshape_safe; ++ int previous_raid_disks; ++ int prev_chunk_sectors; ++ int prev_algo; ++ short generation; /* increments with every reshape */ ++ seqcount_t gen_lock; /* lock against generation changes */ ++ unsigned long reshape_checkpoint; /* Time we last updated ++ * metadata */ ++ long long min_offset_diff; /* minimum difference between ++ * data_offset and ++ * new_data_offset across all ++ * devices. May be negative, ++ * but is closest to zero. ++ */ ++ ++ struct list_head handle_list; /* stripes needing handling */ ++ struct list_head hold_list; /* preread ready stripes */ ++ struct list_head delayed_list; /* stripes that have plugged requests */ ++ struct list_head bitmap_list; /* stripes delaying awaiting bitmap update */ ++ struct bio *retry_read_aligned; /* currently retrying aligned bios */ ++ struct bio *retry_read_aligned_list; /* aligned bios retry list */ ++ atomic_t preread_active_stripes; /* stripes with scheduled io */ ++ atomic_t active_aligned_reads; ++ atomic_t pending_full_writes; /* full write backlog */ ++ int bypass_count; /* bypassed prereads */ ++ int bypass_threshold; /* preread nice */ ++ int skip_copy; /* Don't copy data from bio to stripe cache */ ++ struct list_head *last_hold; /* detect hold_list promotions */ ++ ++ atomic_t reshape_stripes; /* stripes with pending writes for reshape */ ++ /* unfortunately we need two cache names as we temporarily have ++ * two caches. ++ */ ++ int active_name; ++ char cache_name[2][32]; ++ struct kmem_cache *slab_cache; /* for allocating stripes */ ++ struct mutex cache_size_mutex; /* Protect changes to cache size */ ++ ++ int seq_flush, seq_write; ++ int quiesce; ++ ++ int fullsync; /* set to 1 if a full sync is needed, ++ * (fresh device added). ++ * Cleared when a sync completes. ++ */ ++ int recovery_disabled; ++ /* per cpu variables */ ++ struct raid5_percpu { ++ struct page *spare_page; /* Used when checking P/Q in raid6 */ ++ struct flex_array *scribble; /* space for constructing buffer ++ * lists and performing address ++ * conversions ++ */ ++ } __percpu *percpu; ++#ifdef CONFIG_HOTPLUG_CPU ++ struct notifier_block cpu_notify; ++#endif ++ ++ /* ++ * Free stripes pool ++ */ ++ atomic_t active_stripes; ++ struct list_head inactive_list[NR_STRIPE_HASH_LOCKS]; ++ atomic_t empty_inactive_list_nr; ++ struct llist_head released_stripes; ++ wait_queue_head_t wait_for_stripe; ++ wait_queue_head_t wait_for_overlap; ++ unsigned long cache_state; ++#define R5_INACTIVE_BLOCKED 1 /* release of inactive stripes blocked, ++ * waiting for 25% to be free ++ */ ++#define R5_ALLOC_MORE 2 /* It might help to allocate another ++ * stripe. ++ */ ++#define R5_DID_ALLOC 4 /* A stripe was allocated, don't allocate ++ * more until at least one has been ++ * released. This avoids flooding ++ * the cache. ++ */ ++ struct shrinker shrinker; ++ int pool_size; /* number of disks in stripeheads in pool */ ++ spinlock_t device_lock; ++ struct disk_info *disks; ++ ++ /* When taking over an array from a different personality, we store ++ * the new thread here until we fully activate the array. ++ */ ++ struct md_thread *thread; ++ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; ++ struct r5worker_group *worker_groups; ++ int group_cnt; ++ int worker_cnt_per_group; ++}; ++ ++ ++/* ++ * Our supported algorithms ++ */ ++#define ALGORITHM_LEFT_ASYMMETRIC 0 /* Rotating Parity N with Data Restart */ ++#define ALGORITHM_RIGHT_ASYMMETRIC 1 /* Rotating Parity 0 with Data Restart */ ++#define ALGORITHM_LEFT_SYMMETRIC 2 /* Rotating Parity N with Data Continuation */ ++#define ALGORITHM_RIGHT_SYMMETRIC 3 /* Rotating Parity 0 with Data Continuation */ ++ ++/* Define non-rotating (raid4) algorithms. These allow ++ * conversion of raid4 to raid5. ++ */ ++#define ALGORITHM_PARITY_0 4 /* P or P,Q are initial devices */ ++#define ALGORITHM_PARITY_N 5 /* P or P,Q are final devices. */ ++ ++/* DDF RAID6 layouts differ from md/raid6 layouts in two ways. ++ * Firstly, the exact positioning of the parity block is slightly ++ * different between the 'LEFT_*' modes of md and the "_N_*" modes ++ * of DDF. ++ * Secondly, or order of datablocks over which the Q syndrome is computed ++ * is different. ++ * Consequently we have different layouts for DDF/raid6 than md/raid6. ++ * These layouts are from the DDFv1.2 spec. ++ * Interestingly DDFv1.2-Errata-A does not specify N_CONTINUE but ++ * leaves RLQ=3 as 'Vendor Specific' ++ */ ++ ++#define ALGORITHM_ROTATING_ZERO_RESTART 8 /* DDF PRL=6 RLQ=1 */ ++#define ALGORITHM_ROTATING_N_RESTART 9 /* DDF PRL=6 RLQ=2 */ ++#define ALGORITHM_ROTATING_N_CONTINUE 10 /*DDF PRL=6 RLQ=3 */ ++ ++/* For every RAID5 algorithm we define a RAID6 algorithm ++ * with exactly the same layout for data and parity, and ++ * with the Q block always on the last device (N-1). ++ * This allows trivial conversion from RAID5 to RAID6 ++ */ ++#define ALGORITHM_LEFT_ASYMMETRIC_6 16 ++#define ALGORITHM_RIGHT_ASYMMETRIC_6 17 ++#define ALGORITHM_LEFT_SYMMETRIC_6 18 ++#define ALGORITHM_RIGHT_SYMMETRIC_6 19 ++#define ALGORITHM_PARITY_0_6 20 ++#define ALGORITHM_PARITY_N_6 ALGORITHM_PARITY_N ++ ++static inline int algorithm_valid_raid5(int layout) ++{ ++ return (layout >= 0) && ++ (layout <= 5); ++} ++static inline int algorithm_valid_raid6(int layout) ++{ ++ return (layout >= 0 && layout <= 5) ++ || ++ (layout >= 8 && layout <= 10) ++ || ++ (layout >= 16 && layout <= 20); ++} ++ ++static inline int algorithm_is_DDF(int layout) ++{ ++ return layout >= 8 && layout <= 10; ++} ++ ++extern void md_raid5_kick_device(struct r5conf *conf); ++extern int raid5_set_cache_size(struct mddev *mddev, int size); ++#endif +diff -Nur linux-4.1.10.orig/drivers/misc/hwlat_detector.c linux-4.1.10/drivers/misc/hwlat_detector.c +--- linux-4.1.10.orig/drivers/misc/hwlat_detector.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/drivers/misc/hwlat_detector.c 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,1240 @@ +/* + * hwlat_detector.c - A simple Hardware Latency detector. @@ -7651,9 +16142,9 @@ diff -Nur linux-4.1.6.orig/drivers/misc/hwlat_detector.c linux-4.1.6/drivers/mis + +module_init(detector_init); +module_exit(detector_exit); -diff -Nur linux-4.1.6.orig/drivers/misc/Kconfig linux-4.1.6/drivers/misc/Kconfig ---- linux-4.1.6.orig/drivers/misc/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/misc/Kconfig 2015-09-08 23:49:06.082091803 +0200 +diff -Nur linux-4.1.10.orig/drivers/misc/Kconfig linux-4.1.10/drivers/misc/Kconfig +--- linux-4.1.10.orig/drivers/misc/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/misc/Kconfig 2015-10-07 18:00:08.000000000 +0200 @@ -54,6 +54,7 @@ config ATMEL_TCLIB bool "Atmel AT32/AT91 Timer/Counter Library" @@ -7724,9 +16215,9 @@ diff -Nur linux-4.1.6.orig/drivers/misc/Kconfig linux-4.1.6/drivers/misc/Kconfig config PHANTOM tristate "Sensable PHANToM (PCI)" depends on PCI -diff -Nur linux-4.1.6.orig/drivers/misc/Makefile linux-4.1.6/drivers/misc/Makefile ---- linux-4.1.6.orig/drivers/misc/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/misc/Makefile 2015-09-08 23:49:06.082091803 +0200 +diff -Nur linux-4.1.10.orig/drivers/misc/Makefile linux-4.1.10/drivers/misc/Makefile +--- linux-4.1.10.orig/drivers/misc/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/misc/Makefile 2015-10-07 18:00:08.000000000 +0200 @@ -38,6 +38,7 @@ obj-$(CONFIG_HMC6352) += hmc6352.o obj-y += eeprom/ @@ -7735,9 +16226,9 @@ diff -Nur linux-4.1.6.orig/drivers/misc/Makefile linux-4.1.6/drivers/misc/Makefi obj-$(CONFIG_SPEAR13XX_PCIE_GADGET) += spear13xx_pcie_gadget.o obj-$(CONFIG_VMWARE_BALLOON) += vmw_balloon.o obj-$(CONFIG_ARM_CHARLCD) += arm-charlcd.o -diff -Nur linux-4.1.6.orig/drivers/mmc/host/mmci.c linux-4.1.6/drivers/mmc/host/mmci.c ---- linux-4.1.6.orig/drivers/mmc/host/mmci.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/mmc/host/mmci.c 2015-09-08 23:49:06.082091803 +0200 +diff -Nur linux-4.1.10.orig/drivers/mmc/host/mmci.c linux-4.1.10/drivers/mmc/host/mmci.c +--- linux-4.1.10.orig/drivers/mmc/host/mmci.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/mmc/host/mmci.c 2015-10-07 18:00:08.000000000 +0200 @@ -1155,15 +1155,12 @@ struct sg_mapping_iter *sg_miter = &host->sg_miter; struct variant_data *variant = host->variant; @@ -7763,65 +16254,9 @@ diff -Nur linux-4.1.6.orig/drivers/mmc/host/mmci.c linux-4.1.6/drivers/mmc/host/ /* * If we have less than the fifo 'half-full' threshold to transfer, * trigger a PIO interrupt as soon as any data is available. -diff -Nur linux-4.1.6.orig/drivers/mmc/host/sdhci.c linux-4.1.6/drivers/mmc/host/sdhci.c ---- linux-4.1.6.orig/drivers/mmc/host/sdhci.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/mmc/host/sdhci.c 2015-09-08 23:49:06.082091803 +0200 -@@ -2691,6 +2691,31 @@ - return isr ? IRQ_HANDLED : IRQ_NONE; - } - -+#ifdef CONFIG_PREEMPT_RT_BASE -+static irqreturn_t sdhci_rt_irq(int irq, void *dev_id) -+{ -+ irqreturn_t ret; -+ -+ local_bh_disable(); -+ ret = sdhci_irq(irq, dev_id); -+ local_bh_enable(); -+ if (ret == IRQ_WAKE_THREAD) -+ ret = sdhci_thread_irq(irq, dev_id); -+ return ret; -+} -+#endif -+ -+static int sdhci_req_irq(struct sdhci_host *host) -+{ -+#ifdef CONFIG_PREEMPT_RT_BASE -+ return request_threaded_irq(host->irq, NULL, sdhci_rt_irq, -+ IRQF_SHARED, mmc_hostname(host->mmc), host); -+#else -+ return request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq, -+ IRQF_SHARED, mmc_hostname(host->mmc), host); -+#endif -+} -+ - /*****************************************************************************\ - * * - * Suspend/resume * -@@ -2758,9 +2783,7 @@ - } - - if (!device_may_wakeup(mmc_dev(host->mmc))) { -- ret = request_threaded_irq(host->irq, sdhci_irq, -- sdhci_thread_irq, IRQF_SHARED, -- mmc_hostname(host->mmc), host); -+ ret = sdhci_req_irq(host); - if (ret) - return ret; - } else { -@@ -3421,8 +3444,7 @@ - - sdhci_init(host, 0); - -- ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq, -- IRQF_SHARED, mmc_hostname(mmc), host); -+ ret = sdhci_req_irq(host); - if (ret) { - pr_err("%s: Failed to request IRQ %d: %d\n", - mmc_hostname(mmc), host->irq, ret); -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/3com/3c59x.c linux-4.1.6/drivers/net/ethernet/3com/3c59x.c ---- linux-4.1.6.orig/drivers/net/ethernet/3com/3c59x.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/3com/3c59x.c 2015-09-08 23:49:06.086091361 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c linux-4.1.10/drivers/net/ethernet/3com/3c59x.c +--- linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/3com/3c59x.c 2015-10-07 18:00:08.000000000 +0200 @@ -842,9 +842,9 @@ { struct vortex_private *vp = netdev_priv(dev); @@ -7849,9 +16284,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/ethernet/3com/3c59x.c linux-4.1.6/drivers } } -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linux-4.1.6/drivers/net/ethernet/atheros/atl1c/atl1c_main.c ---- linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-09-08 23:49:06.086091361 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +--- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-07 18:00:08.000000000 +0200 @@ -2213,11 +2213,7 @@ } @@ -7865,9 +16300,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linux if (atl1c_tpd_avail(adapter, type) < tpd_req) { /* no enough descriptor, just stop queue */ -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linux-4.1.6/drivers/net/ethernet/atheros/atl1e/atl1e_main.c ---- linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-09-08 23:49:06.086091361 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c +--- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-07 18:00:08.000000000 +0200 @@ -1880,8 +1880,7 @@ return NETDEV_TX_OK; } @@ -7878,9 +16313,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linux if (atl1e_tpd_avail(adapter) < tpd_req) { /* no enough descriptor, just stop queue */ -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.1.6/drivers/net/ethernet/chelsio/cxgb/sge.c ---- linux-4.1.6.orig/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-09-08 23:49:06.086091361 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c +--- linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-07 18:00:08.000000000 +0200 @@ -1664,8 +1664,7 @@ struct cmdQ *q = &sge->cmdQ[qid]; unsigned int credits, pidx, genbit, count, use_sched_skb = 0; @@ -7891,9 +16326,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.1.6/d reclaim_completed_tx(sge, q); -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/freescale/gianfar.c linux-4.1.6/drivers/net/ethernet/freescale/gianfar.c ---- linux-4.1.6.orig/drivers/net/ethernet/freescale/gianfar.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/freescale/gianfar.c 2015-09-08 23:49:06.086091361 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c +--- linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c 2015-10-07 18:00:08.000000000 +0200 @@ -1540,7 +1540,7 @@ if (netif_running(ndev)) { @@ -7947,9 +16382,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/ethernet/freescale/gianfar.c linux-4.1.6/ } netif_dbg(priv, tx_err, dev, "Transmit Error\n"); } -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/neterion/s2io.c linux-4.1.6/drivers/net/ethernet/neterion/s2io.c ---- linux-4.1.6.orig/drivers/net/ethernet/neterion/s2io.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/neterion/s2io.c 2015-09-08 23:49:06.098090029 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c linux-4.1.10/drivers/net/ethernet/neterion/s2io.c +--- linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/neterion/s2io.c 2015-10-07 18:00:08.000000000 +0200 @@ -4084,12 +4084,7 @@ [skb->priority & (MAX_TX_FIFOS - 1)]; fifo = &mac_control->fifos[queue]; @@ -7964,9 +16399,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/ethernet/neterion/s2io.c linux-4.1.6/driv if (sp->config.multiq) { if (__netif_subqueue_stopped(dev, fifo->fifo_no)) { -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c linux-4.1.6/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c ---- linux-4.1.6.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-09-08 23:49:06.302067412 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c +--- linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-07 18:00:08.000000000 +0200 @@ -2137,10 +2137,8 @@ struct pch_gbe_tx_ring *tx_ring = adapter->tx_ring; unsigned long flags; @@ -7980,9 +16415,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c if (unlikely(!PCH_GBE_DESC_UNUSED(tx_ring))) { netif_stop_queue(netdev); spin_unlock_irqrestore(&tx_ring->tx_lock, flags); -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/realtek/8139too.c linux-4.1.6/drivers/net/ethernet/realtek/8139too.c ---- linux-4.1.6.orig/drivers/net/ethernet/realtek/8139too.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/realtek/8139too.c 2015-09-08 23:49:06.306066968 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c linux-4.1.10/drivers/net/ethernet/realtek/8139too.c +--- linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/realtek/8139too.c 2015-10-07 18:00:08.000000000 +0200 @@ -2229,7 +2229,7 @@ struct rtl8139_private *tp = netdev_priv(dev); const int irq = tp->pci_dev->irq; @@ -7992,9 +16427,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/ethernet/realtek/8139too.c linux-4.1.6/dr rtl8139_interrupt(irq, dev); enable_irq(irq); } -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.1.6/drivers/net/ethernet/tehuti/tehuti.c ---- linux-4.1.6.orig/drivers/net/ethernet/tehuti/tehuti.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/tehuti/tehuti.c 2015-09-08 23:49:06.306066968 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c +--- linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c 2015-10-07 18:00:08.000000000 +0200 @@ -1629,13 +1629,8 @@ unsigned long flags; @@ -8011,9 +16446,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.1.6/driv /* build tx descriptor */ BDX_ASSERT(f->m.wptr >= f->m.memsz); /* started with valid wptr */ -diff -Nur linux-4.1.6.orig/drivers/net/rionet.c linux-4.1.6/drivers/net/rionet.c ---- linux-4.1.6.orig/drivers/net/rionet.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/rionet.c 2015-09-08 23:49:06.306066968 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/rionet.c linux-4.1.10/drivers/net/rionet.c +--- linux-4.1.10.orig/drivers/net/rionet.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/rionet.c 2015-10-07 18:00:08.000000000 +0200 @@ -174,11 +174,7 @@ unsigned long flags; int add_num = 1; @@ -8027,9 +16462,9 @@ diff -Nur linux-4.1.6.orig/drivers/net/rionet.c linux-4.1.6/drivers/net/rionet.c if (is_multicast_ether_addr(eth->h_dest)) add_num = nets[rnet->mport->id].nact; -diff -Nur linux-4.1.6.orig/drivers/net/wireless/orinoco/orinoco_usb.c linux-4.1.6/drivers/net/wireless/orinoco/orinoco_usb.c ---- linux-4.1.6.orig/drivers/net/wireless/orinoco/orinoco_usb.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/wireless/orinoco/orinoco_usb.c 2015-09-08 23:49:06.306066968 +0200 +diff -Nur linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c +--- linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-07 18:00:08.000000000 +0200 @@ -697,7 +697,7 @@ while (!ctx->done.done && msecs--) udelay(1000); @@ -8039,10 +16474,10 @@ diff -Nur linux-4.1.6.orig/drivers/net/wireless/orinoco/orinoco_usb.c linux-4.1. ctx->done.done); } break; -diff -Nur linux-4.1.6.orig/drivers/pci/access.c linux-4.1.6/drivers/pci/access.c ---- linux-4.1.6.orig/drivers/pci/access.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/pci/access.c 2015-09-08 23:49:06.310066523 +0200 -@@ -521,7 +521,7 @@ +diff -Nur linux-4.1.10.orig/drivers/pci/access.c linux-4.1.10/drivers/pci/access.c +--- linux-4.1.10.orig/drivers/pci/access.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/pci/access.c 2015-10-07 18:00:08.000000000 +0200 +@@ -580,7 +580,7 @@ WARN_ON(!dev->block_cfg_access); dev->block_cfg_access = 0; @@ -8051,9 +16486,797 @@ diff -Nur linux-4.1.6.orig/drivers/pci/access.c linux-4.1.6/drivers/pci/access.c raw_spin_unlock_irqrestore(&pci_lock, flags); } EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); -diff -Nur linux-4.1.6.orig/drivers/scsi/fcoe/fcoe.c linux-4.1.6/drivers/scsi/fcoe/fcoe.c ---- linux-4.1.6.orig/drivers/scsi/fcoe/fcoe.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/fcoe/fcoe.c 2015-09-08 23:49:06.314066079 +0200 +diff -Nur linux-4.1.10.orig/drivers/pci/access.c.orig linux-4.1.10/drivers/pci/access.c.orig +--- linux-4.1.10.orig/drivers/pci/access.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/drivers/pci/access.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,784 @@ ++#include <linux/delay.h> ++#include <linux/pci.h> ++#include <linux/module.h> ++#include <linux/sched.h> ++#include <linux/slab.h> ++#include <linux/ioport.h> ++#include <linux/wait.h> ++ ++#include "pci.h" ++ ++/* ++ * This interrupt-safe spinlock protects all accesses to PCI ++ * configuration space. ++ */ ++ ++DEFINE_RAW_SPINLOCK(pci_lock); ++ ++/* ++ * Wrappers for all PCI configuration access functions. They just check ++ * alignment, do locking and call the low-level functions pointed to ++ * by pci_dev->ops. ++ */ ++ ++#define PCI_byte_BAD 0 ++#define PCI_word_BAD (pos & 1) ++#define PCI_dword_BAD (pos & 3) ++ ++#define PCI_OP_READ(size,type,len) \ ++int pci_bus_read_config_##size \ ++ (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \ ++{ \ ++ int res; \ ++ unsigned long flags; \ ++ u32 data = 0; \ ++ if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ ++ raw_spin_lock_irqsave(&pci_lock, flags); \ ++ res = bus->ops->read(bus, devfn, pos, len, &data); \ ++ *value = (type)data; \ ++ raw_spin_unlock_irqrestore(&pci_lock, flags); \ ++ return res; \ ++} ++ ++#define PCI_OP_WRITE(size,type,len) \ ++int pci_bus_write_config_##size \ ++ (struct pci_bus *bus, unsigned int devfn, int pos, type value) \ ++{ \ ++ int res; \ ++ unsigned long flags; \ ++ if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ ++ raw_spin_lock_irqsave(&pci_lock, flags); \ ++ res = bus->ops->write(bus, devfn, pos, len, value); \ ++ raw_spin_unlock_irqrestore(&pci_lock, flags); \ ++ return res; \ ++} ++ ++PCI_OP_READ(byte, u8, 1) ++PCI_OP_READ(word, u16, 2) ++PCI_OP_READ(dword, u32, 4) ++PCI_OP_WRITE(byte, u8, 1) ++PCI_OP_WRITE(word, u16, 2) ++PCI_OP_WRITE(dword, u32, 4) ++ ++EXPORT_SYMBOL(pci_bus_read_config_byte); ++EXPORT_SYMBOL(pci_bus_read_config_word); ++EXPORT_SYMBOL(pci_bus_read_config_dword); ++EXPORT_SYMBOL(pci_bus_write_config_byte); ++EXPORT_SYMBOL(pci_bus_write_config_word); ++EXPORT_SYMBOL(pci_bus_write_config_dword); ++ ++int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, ++ int where, int size, u32 *val) ++{ ++ void __iomem *addr; ++ ++ addr = bus->ops->map_bus(bus, devfn, where); ++ if (!addr) { ++ *val = ~0; ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ } ++ ++ if (size == 1) ++ *val = readb(addr); ++ else if (size == 2) ++ *val = readw(addr); ++ else ++ *val = readl(addr); ++ ++ return PCIBIOS_SUCCESSFUL; ++} ++EXPORT_SYMBOL_GPL(pci_generic_config_read); ++ ++int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, ++ int where, int size, u32 val) ++{ ++ void __iomem *addr; ++ ++ addr = bus->ops->map_bus(bus, devfn, where); ++ if (!addr) ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ ++ if (size == 1) ++ writeb(val, addr); ++ else if (size == 2) ++ writew(val, addr); ++ else ++ writel(val, addr); ++ ++ return PCIBIOS_SUCCESSFUL; ++} ++EXPORT_SYMBOL_GPL(pci_generic_config_write); ++ ++int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn, ++ int where, int size, u32 *val) ++{ ++ void __iomem *addr; ++ ++ addr = bus->ops->map_bus(bus, devfn, where & ~0x3); ++ if (!addr) { ++ *val = ~0; ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ } ++ ++ *val = readl(addr); ++ ++ if (size <= 2) ++ *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); ++ ++ return PCIBIOS_SUCCESSFUL; ++} ++EXPORT_SYMBOL_GPL(pci_generic_config_read32); ++ ++int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn, ++ int where, int size, u32 val) ++{ ++ void __iomem *addr; ++ u32 mask, tmp; ++ ++ addr = bus->ops->map_bus(bus, devfn, where & ~0x3); ++ if (!addr) ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ ++ if (size == 4) { ++ writel(val, addr); ++ return PCIBIOS_SUCCESSFUL; ++ } else { ++ mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); ++ } ++ ++ tmp = readl(addr) & mask; ++ tmp |= val << ((where & 0x3) * 8); ++ writel(tmp, addr); ++ ++ return PCIBIOS_SUCCESSFUL; ++} ++EXPORT_SYMBOL_GPL(pci_generic_config_write32); ++ ++/** ++ * pci_bus_set_ops - Set raw operations of pci bus ++ * @bus: pci bus struct ++ * @ops: new raw operations ++ * ++ * Return previous raw operations ++ */ ++struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops) ++{ ++ struct pci_ops *old_ops; ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&pci_lock, flags); ++ old_ops = bus->ops; ++ bus->ops = ops; ++ raw_spin_unlock_irqrestore(&pci_lock, flags); ++ return old_ops; ++} ++EXPORT_SYMBOL(pci_bus_set_ops); ++ ++/** ++ * pci_read_vpd - Read one entry from Vital Product Data ++ * @dev: pci device struct ++ * @pos: offset in vpd space ++ * @count: number of bytes to read ++ * @buf: pointer to where to store result ++ * ++ */ ++ssize_t pci_read_vpd(struct pci_dev *dev, loff_t pos, size_t count, void *buf) ++{ ++ if (!dev->vpd || !dev->vpd->ops) ++ return -ENODEV; ++ return dev->vpd->ops->read(dev, pos, count, buf); ++} ++EXPORT_SYMBOL(pci_read_vpd); ++ ++/** ++ * pci_write_vpd - Write entry to Vital Product Data ++ * @dev: pci device struct ++ * @pos: offset in vpd space ++ * @count: number of bytes to write ++ * @buf: buffer containing write data ++ * ++ */ ++ssize_t pci_write_vpd(struct pci_dev *dev, loff_t pos, size_t count, const void *buf) ++{ ++ if (!dev->vpd || !dev->vpd->ops) ++ return -ENODEV; ++ return dev->vpd->ops->write(dev, pos, count, buf); ++} ++EXPORT_SYMBOL(pci_write_vpd); ++ ++/* ++ * The following routines are to prevent the user from accessing PCI config ++ * space when it's unsafe to do so. Some devices require this during BIST and ++ * we're required to prevent it during D-state transitions. ++ * ++ * We have a bit per device to indicate it's blocked and a global wait queue ++ * for callers to sleep on until devices are unblocked. ++ */ ++static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait); ++ ++static noinline void pci_wait_cfg(struct pci_dev *dev) ++{ ++ DECLARE_WAITQUEUE(wait, current); ++ ++ __add_wait_queue(&pci_cfg_wait, &wait); ++ do { ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ raw_spin_unlock_irq(&pci_lock); ++ schedule(); ++ raw_spin_lock_irq(&pci_lock); ++ } while (dev->block_cfg_access); ++ __remove_wait_queue(&pci_cfg_wait, &wait); ++} ++ ++/* Returns 0 on success, negative values indicate error. */ ++#define PCI_USER_READ_CONFIG(size,type) \ ++int pci_user_read_config_##size \ ++ (struct pci_dev *dev, int pos, type *val) \ ++{ \ ++ int ret = PCIBIOS_SUCCESSFUL; \ ++ u32 data = -1; \ ++ if (PCI_##size##_BAD) \ ++ return -EINVAL; \ ++ raw_spin_lock_irq(&pci_lock); \ ++ if (unlikely(dev->block_cfg_access)) \ ++ pci_wait_cfg(dev); \ ++ ret = dev->bus->ops->read(dev->bus, dev->devfn, \ ++ pos, sizeof(type), &data); \ ++ raw_spin_unlock_irq(&pci_lock); \ ++ *val = (type)data; \ ++ return pcibios_err_to_errno(ret); \ ++} \ ++EXPORT_SYMBOL_GPL(pci_user_read_config_##size); ++ ++/* Returns 0 on success, negative values indicate error. */ ++#define PCI_USER_WRITE_CONFIG(size,type) \ ++int pci_user_write_config_##size \ ++ (struct pci_dev *dev, int pos, type val) \ ++{ \ ++ int ret = PCIBIOS_SUCCESSFUL; \ ++ if (PCI_##size##_BAD) \ ++ return -EINVAL; \ ++ raw_spin_lock_irq(&pci_lock); \ ++ if (unlikely(dev->block_cfg_access)) \ ++ pci_wait_cfg(dev); \ ++ ret = dev->bus->ops->write(dev->bus, dev->devfn, \ ++ pos, sizeof(type), val); \ ++ raw_spin_unlock_irq(&pci_lock); \ ++ return pcibios_err_to_errno(ret); \ ++} \ ++EXPORT_SYMBOL_GPL(pci_user_write_config_##size); ++ ++PCI_USER_READ_CONFIG(byte, u8) ++PCI_USER_READ_CONFIG(word, u16) ++PCI_USER_READ_CONFIG(dword, u32) ++PCI_USER_WRITE_CONFIG(byte, u8) ++PCI_USER_WRITE_CONFIG(word, u16) ++PCI_USER_WRITE_CONFIG(dword, u32) ++ ++/* VPD access through PCI 2.2+ VPD capability */ ++ ++#define PCI_VPD_PCI22_SIZE (PCI_VPD_ADDR_MASK + 1) ++ ++struct pci_vpd_pci22 { ++ struct pci_vpd base; ++ struct mutex lock; ++ u16 flag; ++ bool busy; ++ u8 cap; ++}; ++ ++/* ++ * Wait for last operation to complete. ++ * This code has to spin since there is no other notification from the PCI ++ * hardware. Since the VPD is often implemented by serial attachment to an ++ * EEPROM, it may take many milliseconds to complete. ++ * ++ * Returns 0 on success, negative values indicate error. ++ */ ++static int pci_vpd_pci22_wait(struct pci_dev *dev) ++{ ++ struct pci_vpd_pci22 *vpd = ++ container_of(dev->vpd, struct pci_vpd_pci22, base); ++ unsigned long timeout = jiffies + HZ/20 + 2; ++ u16 status; ++ int ret; ++ ++ if (!vpd->busy) ++ return 0; ++ ++ for (;;) { ++ ret = pci_user_read_config_word(dev, vpd->cap + PCI_VPD_ADDR, ++ &status); ++ if (ret < 0) ++ return ret; ++ ++ if ((status & PCI_VPD_ADDR_F) == vpd->flag) { ++ vpd->busy = false; ++ return 0; ++ } ++ ++ if (time_after(jiffies, timeout)) { ++ dev_printk(KERN_DEBUG, &dev->dev, "vpd r/w failed. This is likely a firmware bug on this device. Contact the card vendor for a firmware update\n"); ++ return -ETIMEDOUT; ++ } ++ if (fatal_signal_pending(current)) ++ return -EINTR; ++ if (!cond_resched()) ++ udelay(10); ++ } ++} ++ ++static ssize_t pci_vpd_pci22_read(struct pci_dev *dev, loff_t pos, size_t count, ++ void *arg) ++{ ++ struct pci_vpd_pci22 *vpd = ++ container_of(dev->vpd, struct pci_vpd_pci22, base); ++ int ret; ++ loff_t end = pos + count; ++ u8 *buf = arg; ++ ++ if (pos < 0 || pos > vpd->base.len || end > vpd->base.len) ++ return -EINVAL; ++ ++ if (mutex_lock_killable(&vpd->lock)) ++ return -EINTR; ++ ++ ret = pci_vpd_pci22_wait(dev); ++ if (ret < 0) ++ goto out; ++ ++ while (pos < end) { ++ u32 val; ++ unsigned int i, skip; ++ ++ ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR, ++ pos & ~3); ++ if (ret < 0) ++ break; ++ vpd->busy = true; ++ vpd->flag = PCI_VPD_ADDR_F; ++ ret = pci_vpd_pci22_wait(dev); ++ if (ret < 0) ++ break; ++ ++ ret = pci_user_read_config_dword(dev, vpd->cap + PCI_VPD_DATA, &val); ++ if (ret < 0) ++ break; ++ ++ skip = pos & 3; ++ for (i = 0; i < sizeof(u32); i++) { ++ if (i >= skip) { ++ *buf++ = val; ++ if (++pos == end) ++ break; ++ } ++ val >>= 8; ++ } ++ } ++out: ++ mutex_unlock(&vpd->lock); ++ return ret ? ret : count; ++} ++ ++static ssize_t pci_vpd_pci22_write(struct pci_dev *dev, loff_t pos, size_t count, ++ const void *arg) ++{ ++ struct pci_vpd_pci22 *vpd = ++ container_of(dev->vpd, struct pci_vpd_pci22, base); ++ const u8 *buf = arg; ++ loff_t end = pos + count; ++ int ret = 0; ++ ++ if (pos < 0 || (pos & 3) || (count & 3) || end > vpd->base.len) ++ return -EINVAL; ++ ++ if (mutex_lock_killable(&vpd->lock)) ++ return -EINTR; ++ ++ ret = pci_vpd_pci22_wait(dev); ++ if (ret < 0) ++ goto out; ++ ++ while (pos < end) { ++ u32 val; ++ ++ val = *buf++; ++ val |= *buf++ << 8; ++ val |= *buf++ << 16; ++ val |= *buf++ << 24; ++ ++ ret = pci_user_write_config_dword(dev, vpd->cap + PCI_VPD_DATA, val); ++ if (ret < 0) ++ break; ++ ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR, ++ pos | PCI_VPD_ADDR_F); ++ if (ret < 0) ++ break; ++ ++ vpd->busy = true; ++ vpd->flag = 0; ++ ret = pci_vpd_pci22_wait(dev); ++ if (ret < 0) ++ break; ++ ++ pos += sizeof(u32); ++ } ++out: ++ mutex_unlock(&vpd->lock); ++ return ret ? ret : count; ++} ++ ++static void pci_vpd_pci22_release(struct pci_dev *dev) ++{ ++ kfree(container_of(dev->vpd, struct pci_vpd_pci22, base)); ++} ++ ++static const struct pci_vpd_ops pci_vpd_pci22_ops = { ++ .read = pci_vpd_pci22_read, ++ .write = pci_vpd_pci22_write, ++ .release = pci_vpd_pci22_release, ++}; ++ ++static ssize_t pci_vpd_f0_read(struct pci_dev *dev, loff_t pos, size_t count, ++ void *arg) ++{ ++ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn)); ++ ssize_t ret; ++ ++ if (!tdev) ++ return -ENODEV; ++ ++ ret = pci_read_vpd(tdev, pos, count, arg); ++ pci_dev_put(tdev); ++ return ret; ++} ++ ++static ssize_t pci_vpd_f0_write(struct pci_dev *dev, loff_t pos, size_t count, ++ const void *arg) ++{ ++ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn)); ++ ssize_t ret; ++ ++ if (!tdev) ++ return -ENODEV; ++ ++ ret = pci_write_vpd(tdev, pos, count, arg); ++ pci_dev_put(tdev); ++ return ret; ++} ++ ++static const struct pci_vpd_ops pci_vpd_f0_ops = { ++ .read = pci_vpd_f0_read, ++ .write = pci_vpd_f0_write, ++ .release = pci_vpd_pci22_release, ++}; ++ ++static int pci_vpd_f0_dev_check(struct pci_dev *dev) ++{ ++ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn)); ++ int ret = 0; ++ ++ if (!tdev) ++ return -ENODEV; ++ if (!tdev->vpd || !tdev->multifunction || ++ dev->class != tdev->class || dev->vendor != tdev->vendor || ++ dev->device != tdev->device) ++ ret = -ENODEV; ++ ++ pci_dev_put(tdev); ++ return ret; ++} ++ ++int pci_vpd_pci22_init(struct pci_dev *dev) ++{ ++ struct pci_vpd_pci22 *vpd; ++ u8 cap; ++ ++ cap = pci_find_capability(dev, PCI_CAP_ID_VPD); ++ if (!cap) ++ return -ENODEV; ++ if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) { ++ int ret = pci_vpd_f0_dev_check(dev); ++ ++ if (ret) ++ return ret; ++ } ++ vpd = kzalloc(sizeof(*vpd), GFP_ATOMIC); ++ if (!vpd) ++ return -ENOMEM; ++ ++ vpd->base.len = PCI_VPD_PCI22_SIZE; ++ if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) ++ vpd->base.ops = &pci_vpd_f0_ops; ++ else ++ vpd->base.ops = &pci_vpd_pci22_ops; ++ mutex_init(&vpd->lock); ++ vpd->cap = cap; ++ vpd->busy = false; ++ dev->vpd = &vpd->base; ++ return 0; ++} ++ ++/** ++ * pci_cfg_access_lock - Lock PCI config reads/writes ++ * @dev: pci device struct ++ * ++ * When access is locked, any userspace reads or writes to config ++ * space and concurrent lock requests will sleep until access is ++ * allowed via pci_cfg_access_unlocked again. ++ */ ++void pci_cfg_access_lock(struct pci_dev *dev) ++{ ++ might_sleep(); ++ ++ raw_spin_lock_irq(&pci_lock); ++ if (dev->block_cfg_access) ++ pci_wait_cfg(dev); ++ dev->block_cfg_access = 1; ++ raw_spin_unlock_irq(&pci_lock); ++} ++EXPORT_SYMBOL_GPL(pci_cfg_access_lock); ++ ++/** ++ * pci_cfg_access_trylock - try to lock PCI config reads/writes ++ * @dev: pci device struct ++ * ++ * Same as pci_cfg_access_lock, but will return 0 if access is ++ * already locked, 1 otherwise. This function can be used from ++ * atomic contexts. ++ */ ++bool pci_cfg_access_trylock(struct pci_dev *dev) ++{ ++ unsigned long flags; ++ bool locked = true; ++ ++ raw_spin_lock_irqsave(&pci_lock, flags); ++ if (dev->block_cfg_access) ++ locked = false; ++ else ++ dev->block_cfg_access = 1; ++ raw_spin_unlock_irqrestore(&pci_lock, flags); ++ ++ return locked; ++} ++EXPORT_SYMBOL_GPL(pci_cfg_access_trylock); ++ ++/** ++ * pci_cfg_access_unlock - Unlock PCI config reads/writes ++ * @dev: pci device struct ++ * ++ * This function allows PCI config accesses to resume. ++ */ ++void pci_cfg_access_unlock(struct pci_dev *dev) ++{ ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&pci_lock, flags); ++ ++ /* This indicates a problem in the caller, but we don't need ++ * to kill them, unlike a double-block above. */ ++ WARN_ON(!dev->block_cfg_access); ++ ++ dev->block_cfg_access = 0; ++ wake_up_all(&pci_cfg_wait); ++ raw_spin_unlock_irqrestore(&pci_lock, flags); ++} ++EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); ++ ++static inline int pcie_cap_version(const struct pci_dev *dev) ++{ ++ return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS; ++} ++ ++bool pcie_cap_has_lnkctl(const struct pci_dev *dev) ++{ ++ int type = pci_pcie_type(dev); ++ ++ return type == PCI_EXP_TYPE_ENDPOINT || ++ type == PCI_EXP_TYPE_LEG_END || ++ type == PCI_EXP_TYPE_ROOT_PORT || ++ type == PCI_EXP_TYPE_UPSTREAM || ++ type == PCI_EXP_TYPE_DOWNSTREAM || ++ type == PCI_EXP_TYPE_PCI_BRIDGE || ++ type == PCI_EXP_TYPE_PCIE_BRIDGE; ++} ++ ++static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev) ++{ ++ int type = pci_pcie_type(dev); ++ ++ return (type == PCI_EXP_TYPE_ROOT_PORT || ++ type == PCI_EXP_TYPE_DOWNSTREAM) && ++ pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT; ++} ++ ++static inline bool pcie_cap_has_rtctl(const struct pci_dev *dev) ++{ ++ int type = pci_pcie_type(dev); ++ ++ return type == PCI_EXP_TYPE_ROOT_PORT || ++ type == PCI_EXP_TYPE_RC_EC; ++} ++ ++static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos) ++{ ++ if (!pci_is_pcie(dev)) ++ return false; ++ ++ switch (pos) { ++ case PCI_EXP_FLAGS: ++ return true; ++ case PCI_EXP_DEVCAP: ++ case PCI_EXP_DEVCTL: ++ case PCI_EXP_DEVSTA: ++ return true; ++ case PCI_EXP_LNKCAP: ++ case PCI_EXP_LNKCTL: ++ case PCI_EXP_LNKSTA: ++ return pcie_cap_has_lnkctl(dev); ++ case PCI_EXP_SLTCAP: ++ case PCI_EXP_SLTCTL: ++ case PCI_EXP_SLTSTA: ++ return pcie_cap_has_sltctl(dev); ++ case PCI_EXP_RTCTL: ++ case PCI_EXP_RTCAP: ++ case PCI_EXP_RTSTA: ++ return pcie_cap_has_rtctl(dev); ++ case PCI_EXP_DEVCAP2: ++ case PCI_EXP_DEVCTL2: ++ case PCI_EXP_LNKCAP2: ++ case PCI_EXP_LNKCTL2: ++ case PCI_EXP_LNKSTA2: ++ return pcie_cap_version(dev) > 1; ++ default: ++ return false; ++ } ++} ++ ++/* ++ * Note that these accessor functions are only for the "PCI Express ++ * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the ++ * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.) ++ */ ++int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val) ++{ ++ int ret; ++ ++ *val = 0; ++ if (pos & 1) ++ return -EINVAL; ++ ++ if (pcie_capability_reg_implemented(dev, pos)) { ++ ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val); ++ /* ++ * Reset *val to 0 if pci_read_config_word() fails, it may ++ * have been written as 0xFFFF if hardware error happens ++ * during pci_read_config_word(). ++ */ ++ if (ret) ++ *val = 0; ++ return ret; ++ } ++ ++ /* ++ * For Functions that do not implement the Slot Capabilities, ++ * Slot Status, and Slot Control registers, these spaces must ++ * be hardwired to 0b, with the exception of the Presence Detect ++ * State bit in the Slot Status register of Downstream Ports, ++ * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8) ++ */ ++ if (pci_is_pcie(dev) && pos == PCI_EXP_SLTSTA && ++ pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) { ++ *val = PCI_EXP_SLTSTA_PDS; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(pcie_capability_read_word); ++ ++int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val) ++{ ++ int ret; ++ ++ *val = 0; ++ if (pos & 3) ++ return -EINVAL; ++ ++ if (pcie_capability_reg_implemented(dev, pos)) { ++ ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val); ++ /* ++ * Reset *val to 0 if pci_read_config_dword() fails, it may ++ * have been written as 0xFFFFFFFF if hardware error happens ++ * during pci_read_config_dword(). ++ */ ++ if (ret) ++ *val = 0; ++ return ret; ++ } ++ ++ if (pci_is_pcie(dev) && pos == PCI_EXP_SLTCTL && ++ pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) { ++ *val = PCI_EXP_SLTSTA_PDS; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(pcie_capability_read_dword); ++ ++int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) ++{ ++ if (pos & 1) ++ return -EINVAL; ++ ++ if (!pcie_capability_reg_implemented(dev, pos)) ++ return 0; ++ ++ return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); ++} ++EXPORT_SYMBOL(pcie_capability_write_word); ++ ++int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val) ++{ ++ if (pos & 3) ++ return -EINVAL; ++ ++ if (!pcie_capability_reg_implemented(dev, pos)) ++ return 0; ++ ++ return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val); ++} ++EXPORT_SYMBOL(pcie_capability_write_dword); ++ ++int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos, ++ u16 clear, u16 set) ++{ ++ int ret; ++ u16 val; ++ ++ ret = pcie_capability_read_word(dev, pos, &val); ++ if (!ret) { ++ val &= ~clear; ++ val |= set; ++ ret = pcie_capability_write_word(dev, pos, val); ++ } ++ ++ return ret; ++} ++EXPORT_SYMBOL(pcie_capability_clear_and_set_word); ++ ++int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos, ++ u32 clear, u32 set) ++{ ++ int ret; ++ u32 val; ++ ++ ret = pcie_capability_read_dword(dev, pos, &val); ++ if (!ret) { ++ val &= ~clear; ++ val |= set; ++ ret = pcie_capability_write_dword(dev, pos, val); ++ } ++ ++ return ret; ++} ++EXPORT_SYMBOL(pcie_capability_clear_and_set_dword); +diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c linux-4.1.10/drivers/scsi/fcoe/fcoe.c +--- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/fcoe/fcoe.c 2015-10-07 18:00:08.000000000 +0200 @@ -1287,7 +1287,7 @@ struct sk_buff *skb; #ifdef CONFIG_SMP @@ -8117,9 +17340,9 @@ diff -Nur linux-4.1.6.orig/drivers/scsi/fcoe/fcoe.c linux-4.1.6/drivers/scsi/fco kfree_skb(skb); } -diff -Nur linux-4.1.6.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.1.6/drivers/scsi/fcoe/fcoe_ctlr.c ---- linux-4.1.6.orig/drivers/scsi/fcoe/fcoe_ctlr.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/fcoe/fcoe_ctlr.c 2015-09-08 23:49:06.314066079 +0200 +diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c +--- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-07 18:00:08.000000000 +0200 @@ -831,7 +831,7 @@ INIT_LIST_HEAD(&del_list); @@ -8138,10 +17361,10 @@ diff -Nur linux-4.1.6.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.1.6/drivers/scs list_for_each_entry_safe(fcf, next, &del_list, list) { /* Removes fcf from current list */ -diff -Nur linux-4.1.6.orig/drivers/scsi/libfc/fc_exch.c linux-4.1.6/drivers/scsi/libfc/fc_exch.c ---- linux-4.1.6.orig/drivers/scsi/libfc/fc_exch.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/libfc/fc_exch.c 2015-09-08 23:49:06.314066079 +0200 -@@ -816,10 +816,10 @@ +diff -Nur linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c linux-4.1.10/drivers/scsi/libfc/fc_exch.c +--- linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/libfc/fc_exch.c 2015-10-07 18:00:08.000000000 +0200 +@@ -814,10 +814,10 @@ } memset(ep, 0, sizeof(*ep)); @@ -8154,9 +17377,9 @@ diff -Nur linux-4.1.6.orig/drivers/scsi/libfc/fc_exch.c linux-4.1.6/drivers/scsi /* peek cache of free slot */ if (pool->left != FC_XID_UNKNOWN) { -diff -Nur linux-4.1.6.orig/drivers/scsi/libsas/sas_ata.c linux-4.1.6/drivers/scsi/libsas/sas_ata.c ---- linux-4.1.6.orig/drivers/scsi/libsas/sas_ata.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/libsas/sas_ata.c 2015-09-08 23:49:06.314066079 +0200 +diff -Nur linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c linux-4.1.10/drivers/scsi/libsas/sas_ata.c +--- linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/libsas/sas_ata.c 2015-10-07 18:00:08.000000000 +0200 @@ -190,7 +190,7 @@ /* TODO: audit callers to ensure they are ready for qc_issue to * unconditionally re-enable interrupts @@ -8175,9 +17398,9 @@ diff -Nur linux-4.1.6.orig/drivers/scsi/libsas/sas_ata.c linux-4.1.6/drivers/scs return ret; } -diff -Nur linux-4.1.6.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.1.6/drivers/scsi/qla2xxx/qla_inline.h ---- linux-4.1.6.orig/drivers/scsi/qla2xxx/qla_inline.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/qla2xxx/qla_inline.h 2015-09-08 23:49:06.314066079 +0200 +diff -Nur linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h +--- linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h 2015-10-07 18:00:08.000000000 +0200 @@ -59,12 +59,12 @@ { unsigned long flags; @@ -8193,9 +17416,9 @@ diff -Nur linux-4.1.6.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.1.6/drivers } static inline uint8_t * -diff -Nur linux-4.1.6.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.1.6/drivers/thermal/x86_pkg_temp_thermal.c ---- linux-4.1.6.orig/drivers/thermal/x86_pkg_temp_thermal.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/thermal/x86_pkg_temp_thermal.c 2015-09-08 23:49:06.314066079 +0200 +diff -Nur linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c +--- linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-07 18:00:08.000000000 +0200 @@ -29,6 +29,7 @@ #include <linux/pm.h> #include <linux/thermal.h> @@ -8298,9 +17521,9 @@ diff -Nur linux-4.1.6.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.1.6/dr for_each_online_cpu(i) cancel_delayed_work_sync( &per_cpu(pkg_temp_thermal_threshold_work, i)); -diff -Nur linux-4.1.6.orig/drivers/tty/serial/8250/8250_core.c linux-4.1.6/drivers/tty/serial/8250/8250_core.c ---- linux-4.1.6.orig/drivers/tty/serial/8250/8250_core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/tty/serial/8250/8250_core.c 2015-09-08 23:49:06.314066079 +0200 +diff -Nur linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c linux-4.1.10/drivers/tty/serial/8250/8250_core.c +--- linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/tty/serial/8250/8250_core.c 2015-10-07 18:00:08.000000000 +0200 @@ -36,6 +36,7 @@ #include <linux/nmi.h> #include <linux/mutex.h> @@ -8336,9 +17559,9 @@ diff -Nur linux-4.1.6.orig/drivers/tty/serial/8250/8250_core.c linux-4.1.6/drive locked = spin_trylock_irqsave(&port->lock, flags); else spin_lock_irqsave(&port->lock, flags); -diff -Nur linux-4.1.6.orig/drivers/tty/serial/amba-pl011.c linux-4.1.6/drivers/tty/serial/amba-pl011.c ---- linux-4.1.6.orig/drivers/tty/serial/amba-pl011.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/tty/serial/amba-pl011.c 2015-09-08 23:49:06.314066079 +0200 +diff -Nur linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c linux-4.1.10/drivers/tty/serial/amba-pl011.c +--- linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/tty/serial/amba-pl011.c 2015-10-07 18:00:08.000000000 +0200 @@ -2000,13 +2000,19 @@ clk_enable(uap->clk); @@ -8372,9 +17595,9 @@ diff -Nur linux-4.1.6.orig/drivers/tty/serial/amba-pl011.c linux-4.1.6/drivers/t clk_disable(uap->clk); } -diff -Nur linux-4.1.6.orig/drivers/tty/serial/omap-serial.c linux-4.1.6/drivers/tty/serial/omap-serial.c ---- linux-4.1.6.orig/drivers/tty/serial/omap-serial.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/tty/serial/omap-serial.c 2015-09-08 23:49:06.314066079 +0200 +diff -Nur linux-4.1.10.orig/drivers/tty/serial/omap-serial.c linux-4.1.10/drivers/tty/serial/omap-serial.c +--- linux-4.1.10.orig/drivers/tty/serial/omap-serial.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/tty/serial/omap-serial.c 2015-10-07 18:00:08.000000000 +0200 @@ -1282,13 +1282,10 @@ pm_runtime_get_sync(up->dev); @@ -8402,9 +17625,9 @@ diff -Nur linux-4.1.6.orig/drivers/tty/serial/omap-serial.c linux-4.1.6/drivers/ } static int __init -diff -Nur linux-4.1.6.orig/drivers/usb/core/hcd.c linux-4.1.6/drivers/usb/core/hcd.c ---- linux-4.1.6.orig/drivers/usb/core/hcd.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/usb/core/hcd.c 2015-09-08 23:49:06.314066079 +0200 +diff -Nur linux-4.1.10.orig/drivers/usb/core/hcd.c linux-4.1.10/drivers/usb/core/hcd.c +--- linux-4.1.10.orig/drivers/usb/core/hcd.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/usb/core/hcd.c 2015-10-07 18:00:08.000000000 +0200 @@ -1684,9 +1684,9 @@ * and no one may trigger the above deadlock situation when * running complete() in tasklet. @@ -8417,9 +17640,9 @@ diff -Nur linux-4.1.6.orig/drivers/usb/core/hcd.c linux-4.1.6/drivers/usb/core/h usb_anchor_resume_wakeups(anchor); atomic_dec(&urb->use_count); -diff -Nur linux-4.1.6.orig/drivers/usb/gadget/function/f_fs.c linux-4.1.6/drivers/usb/gadget/function/f_fs.c ---- linux-4.1.6.orig/drivers/usb/gadget/function/f_fs.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/usb/gadget/function/f_fs.c 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c linux-4.1.10/drivers/usb/gadget/function/f_fs.c +--- linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/usb/gadget/function/f_fs.c 2015-10-07 18:00:08.000000000 +0200 @@ -1405,7 +1405,7 @@ pr_info("%s(): freeing\n", __func__); ffs_data_clear(ffs); @@ -8429,9 +17652,9 @@ diff -Nur linux-4.1.6.orig/drivers/usb/gadget/function/f_fs.c linux-4.1.6/driver kfree(ffs->dev_name); kfree(ffs); } -diff -Nur linux-4.1.6.orig/drivers/usb/gadget/legacy/inode.c linux-4.1.6/drivers/usb/gadget/legacy/inode.c ---- linux-4.1.6.orig/drivers/usb/gadget/legacy/inode.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/usb/gadget/legacy/inode.c 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c linux-4.1.10/drivers/usb/gadget/legacy/inode.c +--- linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/usb/gadget/legacy/inode.c 2015-10-07 18:00:08.000000000 +0200 @@ -345,7 +345,7 @@ spin_unlock_irq (&epdata->dev->lock); @@ -8450,9 +17673,9 @@ diff -Nur linux-4.1.6.orig/drivers/usb/gadget/legacy/inode.c linux-4.1.6/drivers if (epdata->status == -ECONNRESET) epdata->status = -EINTR; } else { -diff -Nur linux-4.1.6.orig/fs/aio.c linux-4.1.6/fs/aio.c ---- linux-4.1.6.orig/fs/aio.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/aio.c 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/fs/aio.c linux-4.1.10/fs/aio.c +--- linux-4.1.10.orig/fs/aio.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/aio.c 2015-10-07 18:00:08.000000000 +0200 @@ -40,6 +40,7 @@ #include <linux/ramfs.h> #include <linux/percpu-refcount.h> @@ -8528,9 +17751,9 @@ diff -Nur linux-4.1.6.orig/fs/aio.c linux-4.1.6/fs/aio.c static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) { unsigned i, new_nr; -diff -Nur linux-4.1.6.orig/fs/autofs4/autofs_i.h linux-4.1.6/fs/autofs4/autofs_i.h ---- linux-4.1.6.orig/fs/autofs4/autofs_i.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/autofs4/autofs_i.h 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/fs/autofs4/autofs_i.h linux-4.1.10/fs/autofs4/autofs_i.h +--- linux-4.1.10.orig/fs/autofs4/autofs_i.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/autofs4/autofs_i.h 2015-10-07 18:00:08.000000000 +0200 @@ -34,6 +34,7 @@ #include <linux/sched.h> #include <linux/mount.h> @@ -8539,9 +17762,9 @@ diff -Nur linux-4.1.6.orig/fs/autofs4/autofs_i.h linux-4.1.6/fs/autofs4/autofs_i #include <asm/current.h> #include <asm/uaccess.h> -diff -Nur linux-4.1.6.orig/fs/autofs4/expire.c linux-4.1.6/fs/autofs4/expire.c ---- linux-4.1.6.orig/fs/autofs4/expire.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/autofs4/expire.c 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/fs/autofs4/expire.c linux-4.1.10/fs/autofs4/expire.c +--- linux-4.1.10.orig/fs/autofs4/expire.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/autofs4/expire.c 2015-10-07 18:00:08.000000000 +0200 @@ -150,7 +150,7 @@ parent = p->d_parent; if (!spin_trylock(&parent->d_lock)) { @@ -8551,9 +17774,9 @@ diff -Nur linux-4.1.6.orig/fs/autofs4/expire.c linux-4.1.6/fs/autofs4/expire.c goto relock; } spin_unlock(&p->d_lock); -diff -Nur linux-4.1.6.orig/fs/buffer.c linux-4.1.6/fs/buffer.c ---- linux-4.1.6.orig/fs/buffer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/buffer.c 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/fs/buffer.c linux-4.1.10/fs/buffer.c +--- linux-4.1.10.orig/fs/buffer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/buffer.c 2015-10-07 18:00:08.000000000 +0200 @@ -301,8 +301,7 @@ * decide that the page is now completely done. */ @@ -8621,9 +17844,9 @@ diff -Nur linux-4.1.6.orig/fs/buffer.c linux-4.1.6/fs/buffer.c preempt_disable(); __this_cpu_inc(bh_accounting.nr); recalc_bh_state(); -diff -Nur linux-4.1.6.orig/fs/dcache.c linux-4.1.6/fs/dcache.c ---- linux-4.1.6.orig/fs/dcache.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/dcache.c 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/fs/dcache.c linux-4.1.10/fs/dcache.c +--- linux-4.1.10.orig/fs/dcache.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/dcache.c 2015-10-07 18:00:08.000000000 +0200 @@ -19,6 +19,7 @@ #include <linux/mm.h> #include <linux/fs.h> @@ -8650,9 +17873,9 @@ diff -Nur linux-4.1.6.orig/fs/dcache.c linux-4.1.6/fs/dcache.c goto again; } dentry->d_flags &= ~DCACHE_CANT_MOUNT; -diff -Nur linux-4.1.6.orig/fs/eventpoll.c linux-4.1.6/fs/eventpoll.c ---- linux-4.1.6.orig/fs/eventpoll.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/eventpoll.c 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/fs/eventpoll.c linux-4.1.10/fs/eventpoll.c +--- linux-4.1.10.orig/fs/eventpoll.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/eventpoll.c 2015-10-07 18:00:08.000000000 +0200 @@ -505,12 +505,12 @@ */ static void ep_poll_safewake(wait_queue_head_t *wq) @@ -8668,9 +17891,9 @@ diff -Nur linux-4.1.6.orig/fs/eventpoll.c linux-4.1.6/fs/eventpoll.c } static void ep_remove_wait_queue(struct eppoll_entry *pwq) -diff -Nur linux-4.1.6.orig/fs/exec.c linux-4.1.6/fs/exec.c ---- linux-4.1.6.orig/fs/exec.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/exec.c 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/fs/exec.c linux-4.1.10/fs/exec.c +--- linux-4.1.10.orig/fs/exec.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/exec.c 2015-10-07 18:00:08.000000000 +0200 @@ -859,12 +859,14 @@ } } @@ -8686,9 +17909,9 @@ diff -Nur linux-4.1.6.orig/fs/exec.c linux-4.1.6/fs/exec.c task_unlock(tsk); if (old_mm) { up_read(&old_mm->mmap_sem); -diff -Nur linux-4.1.6.orig/fs/jbd/checkpoint.c linux-4.1.6/fs/jbd/checkpoint.c ---- linux-4.1.6.orig/fs/jbd/checkpoint.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/jbd/checkpoint.c 2015-09-08 23:49:06.318065636 +0200 +diff -Nur linux-4.1.10.orig/fs/jbd/checkpoint.c linux-4.1.10/fs/jbd/checkpoint.c +--- linux-4.1.10.orig/fs/jbd/checkpoint.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/jbd/checkpoint.c 2015-10-07 18:00:08.000000000 +0200 @@ -129,6 +129,8 @@ if (journal->j_flags & JFS_ABORT) return; @@ -8698,9 +17921,9 @@ diff -Nur linux-4.1.6.orig/fs/jbd/checkpoint.c linux-4.1.6/fs/jbd/checkpoint.c mutex_lock(&journal->j_checkpoint_mutex); /* -diff -Nur linux-4.1.6.orig/fs/jbd2/checkpoint.c linux-4.1.6/fs/jbd2/checkpoint.c ---- linux-4.1.6.orig/fs/jbd2/checkpoint.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/jbd2/checkpoint.c 2015-09-08 23:49:06.322065194 +0200 +diff -Nur linux-4.1.10.orig/fs/jbd2/checkpoint.c linux-4.1.10/fs/jbd2/checkpoint.c +--- linux-4.1.10.orig/fs/jbd2/checkpoint.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/jbd2/checkpoint.c 2015-10-07 18:00:08.000000000 +0200 @@ -116,6 +116,8 @@ nblocks = jbd2_space_needed(journal); while (jbd2_log_space_left(journal) < nblocks) { @@ -8710,9 +17933,9 @@ diff -Nur linux-4.1.6.orig/fs/jbd2/checkpoint.c linux-4.1.6/fs/jbd2/checkpoint.c mutex_lock(&journal->j_checkpoint_mutex); /* -diff -Nur linux-4.1.6.orig/fs/namespace.c linux-4.1.6/fs/namespace.c ---- linux-4.1.6.orig/fs/namespace.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/namespace.c 2015-09-08 23:49:06.322065194 +0200 +diff -Nur linux-4.1.10.orig/fs/namespace.c linux-4.1.10/fs/namespace.c +--- linux-4.1.10.orig/fs/namespace.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/namespace.c 2015-10-07 18:00:08.000000000 +0200 @@ -14,6 +14,7 @@ #include <linux/mnt_namespace.h> #include <linux/user_namespace.h> @@ -8735,9 +17958,9 @@ diff -Nur linux-4.1.6.orig/fs/namespace.c linux-4.1.6/fs/namespace.c /* * After the slowpath clears MNT_WRITE_HOLD, mnt_is_readonly will * be set to match its requirements. So we must not load that until -diff -Nur linux-4.1.6.orig/fs/ntfs/aops.c linux-4.1.6/fs/ntfs/aops.c ---- linux-4.1.6.orig/fs/ntfs/aops.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/ntfs/aops.c 2015-09-08 23:49:06.322065194 +0200 +diff -Nur linux-4.1.10.orig/fs/ntfs/aops.c linux-4.1.10/fs/ntfs/aops.c +--- linux-4.1.10.orig/fs/ntfs/aops.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/ntfs/aops.c 2015-10-07 18:00:08.000000000 +0200 @@ -107,8 +107,7 @@ "0x%llx.", (unsigned long long)bh->b_blocknr); } @@ -8785,9 +18008,9 @@ diff -Nur linux-4.1.6.orig/fs/ntfs/aops.c linux-4.1.6/fs/ntfs/aops.c } /** -diff -Nur linux-4.1.6.orig/fs/timerfd.c linux-4.1.6/fs/timerfd.c ---- linux-4.1.6.orig/fs/timerfd.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/timerfd.c 2015-09-08 23:49:06.322065194 +0200 +diff -Nur linux-4.1.10.orig/fs/timerfd.c linux-4.1.10/fs/timerfd.c +--- linux-4.1.10.orig/fs/timerfd.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/timerfd.c 2015-10-07 18:00:08.000000000 +0200 @@ -450,7 +450,10 @@ break; } @@ -8800,9 +18023,234 @@ diff -Nur linux-4.1.6.orig/fs/timerfd.c linux-4.1.6/fs/timerfd.c } /* -diff -Nur linux-4.1.6.orig/include/acpi/platform/aclinux.h linux-4.1.6/include/acpi/platform/aclinux.h ---- linux-4.1.6.orig/include/acpi/platform/aclinux.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/acpi/platform/aclinux.h 2015-09-08 23:49:06.322065194 +0200 +diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.c linux-4.1.10/fs/xfs/xfs_inode.c +--- linux-4.1.10.orig/fs/xfs/xfs_inode.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/xfs/xfs_inode.c 2015-10-07 18:00:08.000000000 +0200 +@@ -164,7 +164,7 @@ + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); +- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); ++ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + + if (lock_flags & XFS_IOLOCK_EXCL) + mrupdate_nested(&ip->i_iolock, XFS_IOLOCK_DEP(lock_flags)); +@@ -212,7 +212,7 @@ + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); +- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); ++ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + + if (lock_flags & XFS_IOLOCK_EXCL) { + if (!mrtryupdate(&ip->i_iolock)) +@@ -281,7 +281,7 @@ + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); +- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); ++ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + ASSERT(lock_flags != 0); + + if (lock_flags & XFS_IOLOCK_EXCL) +@@ -364,30 +364,38 @@ + + /* + * Bump the subclass so xfs_lock_inodes() acquires each lock with a different +- * value. This shouldn't be called for page fault locking, but we also need to +- * ensure we don't overrun the number of lockdep subclasses for the iolock or +- * mmaplock as that is limited to 12 by the mmap lock lockdep annotations. ++ * value. This can be called for any type of inode lock combination, including ++ * parent locking. Care must be taken to ensure we don't overrun the subclass ++ * storage fields in the class mask we build. + */ + static inline int + xfs_lock_inumorder(int lock_mode, int subclass) + { ++ int class = 0; ++ ++ ASSERT(!(lock_mode & (XFS_ILOCK_PARENT | XFS_ILOCK_RTBITMAP | ++ XFS_ILOCK_RTSUM))); ++ + if (lock_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL)) { +- ASSERT(subclass + XFS_LOCK_INUMORDER < +- (1 << (XFS_MMAPLOCK_SHIFT - XFS_IOLOCK_SHIFT))); +- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << XFS_IOLOCK_SHIFT; ++ ASSERT(subclass <= XFS_IOLOCK_MAX_SUBCLASS); ++ ASSERT(subclass + XFS_IOLOCK_PARENT_VAL < ++ MAX_LOCKDEP_SUBCLASSES); ++ class += subclass << XFS_IOLOCK_SHIFT; ++ if (lock_mode & XFS_IOLOCK_PARENT) ++ class += XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT; + } + + if (lock_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL)) { +- ASSERT(subclass + XFS_LOCK_INUMORDER < +- (1 << (XFS_ILOCK_SHIFT - XFS_MMAPLOCK_SHIFT))); +- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << +- XFS_MMAPLOCK_SHIFT; ++ ASSERT(subclass <= XFS_MMAPLOCK_MAX_SUBCLASS); ++ class += subclass << XFS_MMAPLOCK_SHIFT; + } + +- if (lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)) +- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << XFS_ILOCK_SHIFT; ++ if (lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)) { ++ ASSERT(subclass <= XFS_ILOCK_MAX_SUBCLASS); ++ class += subclass << XFS_ILOCK_SHIFT; ++ } + +- return lock_mode; ++ return (lock_mode & ~XFS_LOCK_SUBCLASS_MASK) | class; + } + + /* +@@ -399,6 +407,11 @@ + * transaction (such as truncate). This can result in deadlock since the long + * running trans might need to wait for the inode we just locked in order to + * push the tail and free space in the log. ++ * ++ * xfs_lock_inodes() can only be used to lock one type of lock at a time - ++ * the iolock, the mmaplock or the ilock, but not more than one at a time. If we ++ * lock more than one at a time, lockdep will report false positives saying we ++ * have violated locking orders. + */ + void + xfs_lock_inodes( +@@ -409,8 +422,29 @@ + int attempts = 0, i, j, try_lock; + xfs_log_item_t *lp; + +- /* currently supports between 2 and 5 inodes */ ++ /* ++ * Currently supports between 2 and 5 inodes with exclusive locking. We ++ * support an arbitrary depth of locking here, but absolute limits on ++ * inodes depend on the the type of locking and the limits placed by ++ * lockdep annotations in xfs_lock_inumorder. These are all checked by ++ * the asserts. ++ */ + ASSERT(ips && inodes >= 2 && inodes <= 5); ++ ASSERT(lock_mode & (XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL | ++ XFS_ILOCK_EXCL)); ++ ASSERT(!(lock_mode & (XFS_IOLOCK_SHARED | XFS_MMAPLOCK_SHARED | ++ XFS_ILOCK_SHARED))); ++ ASSERT(!(lock_mode & XFS_IOLOCK_EXCL) || ++ inodes <= XFS_IOLOCK_MAX_SUBCLASS + 1); ++ ASSERT(!(lock_mode & XFS_MMAPLOCK_EXCL) || ++ inodes <= XFS_MMAPLOCK_MAX_SUBCLASS + 1); ++ ASSERT(!(lock_mode & XFS_ILOCK_EXCL) || ++ inodes <= XFS_ILOCK_MAX_SUBCLASS + 1); ++ ++ if (lock_mode & XFS_IOLOCK_EXCL) { ++ ASSERT(!(lock_mode & (XFS_MMAPLOCK_EXCL | XFS_ILOCK_EXCL))); ++ } else if (lock_mode & XFS_MMAPLOCK_EXCL) ++ ASSERT(!(lock_mode & XFS_ILOCK_EXCL)); + + try_lock = 0; + i = 0; +diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.h linux-4.1.10/fs/xfs/xfs_inode.h +--- linux-4.1.10.orig/fs/xfs/xfs_inode.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/xfs/xfs_inode.h 2015-10-07 18:00:08.000000000 +0200 +@@ -284,9 +284,9 @@ + * Flags for lockdep annotations. + * + * XFS_LOCK_PARENT - for directory operations that require locking a +- * parent directory inode and a child entry inode. The parent gets locked +- * with this flag so it gets a lockdep subclass of 1 and the child entry +- * lock will have a lockdep subclass of 0. ++ * parent directory inode and a child entry inode. IOLOCK requires nesting, ++ * MMAPLOCK does not support this class, ILOCK requires a single subclass ++ * to differentiate parent from child. + * + * XFS_LOCK_RTBITMAP/XFS_LOCK_RTSUM - the realtime device bitmap and summary + * inodes do not participate in the normal lock order, and thus have their +@@ -295,30 +295,63 @@ + * XFS_LOCK_INUMORDER - for locking several inodes at the some time + * with xfs_lock_inodes(). This flag is used as the starting subclass + * and each subsequent lock acquired will increment the subclass by one. +- * So the first lock acquired will have a lockdep subclass of 4, the +- * second lock will have a lockdep subclass of 5, and so on. It is +- * the responsibility of the class builder to shift this to the correct +- * portion of the lock_mode lockdep mask. ++ * However, MAX_LOCKDEP_SUBCLASSES == 8, which means we are greatly ++ * limited to the subclasses we can represent via nesting. We need at least ++ * 5 inodes nest depth for the ILOCK through rename, and we also have to support ++ * XFS_ILOCK_PARENT, which gives 6 subclasses. Then we have XFS_ILOCK_RTBITMAP ++ * and XFS_ILOCK_RTSUM, which are another 2 unique subclasses, so that's all ++ * 8 subclasses supported by lockdep. ++ * ++ * This also means we have to number the sub-classes in the lowest bits of ++ * the mask we keep, and we have to ensure we never exceed 3 bits of lockdep ++ * mask and we can't use bit-masking to build the subclasses. What a mess. ++ * ++ * Bit layout: ++ * ++ * Bit Lock Region ++ * 16-19 XFS_IOLOCK_SHIFT dependencies ++ * 20-23 XFS_MMAPLOCK_SHIFT dependencies ++ * 24-31 XFS_ILOCK_SHIFT dependencies ++ * ++ * IOLOCK values ++ * ++ * 0-3 subclass value ++ * 4-7 PARENT subclass values ++ * ++ * MMAPLOCK values ++ * ++ * 0-3 subclass value ++ * 4-7 unused ++ * ++ * ILOCK values ++ * 0-4 subclass values ++ * 5 PARENT subclass (not nestable) ++ * 6 RTBITMAP subclass (not nestable) ++ * 7 RTSUM subclass (not nestable) ++ * + */ +-#define XFS_LOCK_PARENT 1 +-#define XFS_LOCK_RTBITMAP 2 +-#define XFS_LOCK_RTSUM 3 +-#define XFS_LOCK_INUMORDER 4 +- +-#define XFS_IOLOCK_SHIFT 16 +-#define XFS_IOLOCK_PARENT (XFS_LOCK_PARENT << XFS_IOLOCK_SHIFT) ++#define XFS_IOLOCK_SHIFT 16 ++#define XFS_IOLOCK_PARENT_VAL 4 ++#define XFS_IOLOCK_MAX_SUBCLASS (XFS_IOLOCK_PARENT_VAL - 1) ++#define XFS_IOLOCK_DEP_MASK 0x000f0000 ++#define XFS_IOLOCK_PARENT (XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT) + +-#define XFS_MMAPLOCK_SHIFT 20 ++#define XFS_MMAPLOCK_SHIFT 20 ++#define XFS_MMAPLOCK_NUMORDER 0 ++#define XFS_MMAPLOCK_MAX_SUBCLASS 3 ++#define XFS_MMAPLOCK_DEP_MASK 0x00f00000 + +-#define XFS_ILOCK_SHIFT 24 +-#define XFS_ILOCK_PARENT (XFS_LOCK_PARENT << XFS_ILOCK_SHIFT) +-#define XFS_ILOCK_RTBITMAP (XFS_LOCK_RTBITMAP << XFS_ILOCK_SHIFT) +-#define XFS_ILOCK_RTSUM (XFS_LOCK_RTSUM << XFS_ILOCK_SHIFT) ++#define XFS_ILOCK_SHIFT 24 ++#define XFS_ILOCK_PARENT_VAL 5 ++#define XFS_ILOCK_MAX_SUBCLASS (XFS_ILOCK_PARENT_VAL - 1) ++#define XFS_ILOCK_RTBITMAP_VAL 6 ++#define XFS_ILOCK_RTSUM_VAL 7 ++#define XFS_ILOCK_DEP_MASK 0xff000000 ++#define XFS_ILOCK_PARENT (XFS_ILOCK_PARENT_VAL << XFS_ILOCK_SHIFT) ++#define XFS_ILOCK_RTBITMAP (XFS_ILOCK_RTBITMAP_VAL << XFS_ILOCK_SHIFT) ++#define XFS_ILOCK_RTSUM (XFS_ILOCK_RTSUM_VAL << XFS_ILOCK_SHIFT) + +-#define XFS_IOLOCK_DEP_MASK 0x000f0000 +-#define XFS_MMAPLOCK_DEP_MASK 0x00f00000 +-#define XFS_ILOCK_DEP_MASK 0xff000000 +-#define XFS_LOCK_DEP_MASK (XFS_IOLOCK_DEP_MASK | \ ++#define XFS_LOCK_SUBCLASS_MASK (XFS_IOLOCK_DEP_MASK | \ + XFS_MMAPLOCK_DEP_MASK | \ + XFS_ILOCK_DEP_MASK) + +diff -Nur linux-4.1.10.orig/include/acpi/platform/aclinux.h linux-4.1.10/include/acpi/platform/aclinux.h +--- linux-4.1.10.orig/include/acpi/platform/aclinux.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/acpi/platform/aclinux.h 2015-10-07 18:00:08.000000000 +0200 @@ -123,6 +123,7 @@ #define acpi_cache_t struct kmem_cache @@ -8832,9 +18280,9 @@ diff -Nur linux-4.1.6.orig/include/acpi/platform/aclinux.h linux-4.1.6/include/a /* * OSL interfaces used by debugger/disassembler */ -diff -Nur linux-4.1.6.orig/include/asm-generic/bug.h linux-4.1.6/include/asm-generic/bug.h ---- linux-4.1.6.orig/include/asm-generic/bug.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/asm-generic/bug.h 2015-09-08 23:49:06.322065194 +0200 +diff -Nur linux-4.1.10.orig/include/asm-generic/bug.h linux-4.1.10/include/asm-generic/bug.h +--- linux-4.1.10.orig/include/asm-generic/bug.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/asm-generic/bug.h 2015-10-07 18:00:08.000000000 +0200 @@ -206,6 +206,20 @@ # define WARN_ON_SMP(x) ({0;}) #endif @@ -8856,9 +18304,9 @@ diff -Nur linux-4.1.6.orig/include/asm-generic/bug.h linux-4.1.6/include/asm-gen #endif /* __ASSEMBLY__ */ #endif -diff -Nur linux-4.1.6.orig/include/asm-generic/futex.h linux-4.1.6/include/asm-generic/futex.h ---- linux-4.1.6.orig/include/asm-generic/futex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/asm-generic/futex.h 2015-09-08 23:49:06.322065194 +0200 +diff -Nur linux-4.1.10.orig/include/asm-generic/futex.h linux-4.1.10/include/asm-generic/futex.h +--- linux-4.1.10.orig/include/asm-generic/futex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/asm-generic/futex.h 2015-10-07 18:00:08.000000000 +0200 @@ -8,8 +8,7 @@ #ifndef CONFIG_SMP /* @@ -8901,9 +18349,9 @@ diff -Nur linux-4.1.6.orig/include/asm-generic/futex.h linux-4.1.6/include/asm-g return 0; } -diff -Nur linux-4.1.6.orig/include/linux/blkdev.h linux-4.1.6/include/linux/blkdev.h ---- linux-4.1.6.orig/include/linux/blkdev.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/blkdev.h 2015-09-08 23:49:06.530042129 +0200 +diff -Nur linux-4.1.10.orig/include/linux/blkdev.h linux-4.1.10/include/linux/blkdev.h +--- linux-4.1.10.orig/include/linux/blkdev.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/blkdev.h 2015-10-07 18:00:08.000000000 +0200 @@ -101,6 +101,7 @@ struct list_head queuelist; union { @@ -8921,9 +18369,9 @@ diff -Nur linux-4.1.6.orig/include/linux/blkdev.h linux-4.1.6/include/linux/blkd struct percpu_ref mq_usage_counter; struct list_head all_q_node; -diff -Nur linux-4.1.6.orig/include/linux/blk-mq.h linux-4.1.6/include/linux/blk-mq.h ---- linux-4.1.6.orig/include/linux/blk-mq.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/blk-mq.h 2015-09-08 23:49:06.322065194 +0200 +diff -Nur linux-4.1.10.orig/include/linux/blk-mq.h linux-4.1.10/include/linux/blk-mq.h +--- linux-4.1.10.orig/include/linux/blk-mq.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/blk-mq.h 2015-10-07 18:00:08.000000000 +0200 @@ -202,6 +202,7 @@ struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); @@ -8932,9 +18380,9 @@ diff -Nur linux-4.1.6.orig/include/linux/blk-mq.h linux-4.1.6/include/linux/blk- int blk_mq_request_started(struct request *rq); void blk_mq_start_request(struct request *rq); -diff -Nur linux-4.1.6.orig/include/linux/bottom_half.h linux-4.1.6/include/linux/bottom_half.h ---- linux-4.1.6.orig/include/linux/bottom_half.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/bottom_half.h 2015-09-08 23:49:06.530042129 +0200 +diff -Nur linux-4.1.10.orig/include/linux/bottom_half.h linux-4.1.10/include/linux/bottom_half.h +--- linux-4.1.10.orig/include/linux/bottom_half.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/bottom_half.h 2015-10-07 18:00:08.000000000 +0200 @@ -4,6 +4,17 @@ #include <linux/preempt.h> #include <linux/preempt_mask.h> @@ -8960,9 +18408,9 @@ diff -Nur linux-4.1.6.orig/include/linux/bottom_half.h linux-4.1.6/include/linux +#endif #endif /* _LINUX_BH_H */ -diff -Nur linux-4.1.6.orig/include/linux/buffer_head.h linux-4.1.6/include/linux/buffer_head.h ---- linux-4.1.6.orig/include/linux/buffer_head.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/buffer_head.h 2015-09-08 23:49:06.530042129 +0200 +diff -Nur linux-4.1.10.orig/include/linux/buffer_head.h linux-4.1.10/include/linux/buffer_head.h +--- linux-4.1.10.orig/include/linux/buffer_head.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/buffer_head.h 2015-10-07 18:00:08.000000000 +0200 @@ -75,8 +75,52 @@ struct address_space *b_assoc_map; /* mapping this buffer is associated with */ @@ -9016,9 +18464,9 @@ diff -Nur linux-4.1.6.orig/include/linux/buffer_head.h linux-4.1.6/include/linux /* * macro tricks to expand the set_buffer_foo(), clear_buffer_foo() * and buffer_foo() functions. -diff -Nur linux-4.1.6.orig/include/linux/cgroup.h linux-4.1.6/include/linux/cgroup.h ---- linux-4.1.6.orig/include/linux/cgroup.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/cgroup.h 2015-09-08 23:49:06.534041686 +0200 +diff -Nur linux-4.1.10.orig/include/linux/cgroup.h linux-4.1.10/include/linux/cgroup.h +--- linux-4.1.10.orig/include/linux/cgroup.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/cgroup.h 2015-10-07 18:00:08.000000000 +0200 @@ -22,6 +22,7 @@ #include <linux/seq_file.h> #include <linux/kernfs.h> @@ -9035,9 +18483,9 @@ diff -Nur linux-4.1.6.orig/include/linux/cgroup.h linux-4.1.6/include/linux/cgro }; /* bits in struct cgroup_subsys_state flags field */ -diff -Nur linux-4.1.6.orig/include/linux/completion.h linux-4.1.6/include/linux/completion.h ---- linux-4.1.6.orig/include/linux/completion.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/completion.h 2015-09-08 23:49:06.534041686 +0200 +diff -Nur linux-4.1.10.orig/include/linux/completion.h linux-4.1.10/include/linux/completion.h +--- linux-4.1.10.orig/include/linux/completion.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/completion.h 2015-10-07 18:00:08.000000000 +0200 @@ -7,8 +7,7 @@ * Atomic wait-for-completion handler data structures. * See kernel/sched/completion.c for details. @@ -9071,9 +18519,9 @@ diff -Nur linux-4.1.6.orig/include/linux/completion.h linux-4.1.6/include/linux/ } /** -diff -Nur linux-4.1.6.orig/include/linux/cpu.h linux-4.1.6/include/linux/cpu.h ---- linux-4.1.6.orig/include/linux/cpu.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/cpu.h 2015-09-08 23:49:06.534041686 +0200 +diff -Nur linux-4.1.10.orig/include/linux/cpu.h linux-4.1.10/include/linux/cpu.h +--- linux-4.1.10.orig/include/linux/cpu.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/cpu.h 2015-10-07 18:00:08.000000000 +0200 @@ -231,6 +231,8 @@ extern void put_online_cpus(void); extern void cpu_hotplug_disable(void); @@ -9092,9 +18540,9 @@ diff -Nur linux-4.1.6.orig/include/linux/cpu.h linux-4.1.6/include/linux/cpu.h #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) #define __hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) /* These aren't inline functions due to a GCC bug. */ -diff -Nur linux-4.1.6.orig/include/linux/delay.h linux-4.1.6/include/linux/delay.h ---- linux-4.1.6.orig/include/linux/delay.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/delay.h 2015-09-08 23:49:06.534041686 +0200 +diff -Nur linux-4.1.10.orig/include/linux/delay.h linux-4.1.10/include/linux/delay.h +--- linux-4.1.10.orig/include/linux/delay.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/delay.h 2015-10-07 18:00:08.000000000 +0200 @@ -52,4 +52,10 @@ msleep(seconds * 1000); } @@ -9106,9 +18554,9 @@ diff -Nur linux-4.1.6.orig/include/linux/delay.h linux-4.1.6/include/linux/delay +#endif + #endif /* defined(_LINUX_DELAY_H) */ -diff -Nur linux-4.1.6.orig/include/linux/ftrace_event.h linux-4.1.6/include/linux/ftrace_event.h ---- linux-4.1.6.orig/include/linux/ftrace_event.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/ftrace_event.h 2015-09-08 23:49:06.534041686 +0200 +diff -Nur linux-4.1.10.orig/include/linux/ftrace_event.h linux-4.1.10/include/linux/ftrace_event.h +--- linux-4.1.10.orig/include/linux/ftrace_event.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/ftrace_event.h 2015-10-07 18:00:08.000000000 +0200 @@ -66,6 +66,9 @@ unsigned char flags; unsigned char preempt_count; @@ -9119,9 +18567,9 @@ diff -Nur linux-4.1.6.orig/include/linux/ftrace_event.h linux-4.1.6/include/linu }; #define FTRACE_MAX_EVENT \ -diff -Nur linux-4.1.6.orig/include/linux/highmem.h linux-4.1.6/include/linux/highmem.h ---- linux-4.1.6.orig/include/linux/highmem.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/highmem.h 2015-09-08 23:49:06.534041686 +0200 +diff -Nur linux-4.1.10.orig/include/linux/highmem.h linux-4.1.10/include/linux/highmem.h +--- linux-4.1.10.orig/include/linux/highmem.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/highmem.h 2015-10-07 18:00:08.000000000 +0200 @@ -7,6 +7,7 @@ #include <linux/mm.h> #include <linux/uaccess.h> @@ -9134,7 +18582,7 @@ diff -Nur linux-4.1.6.orig/include/linux/highmem.h linux-4.1.6/include/linux/hig static inline void *kmap_atomic(struct page *page) { -+ preempt_disable(); ++ preempt_disable_nort(); pagefault_disable(); return page_address(page); } @@ -9142,7 +18590,7 @@ diff -Nur linux-4.1.6.orig/include/linux/highmem.h linux-4.1.6/include/linux/hig static inline void __kunmap_atomic(void *addr) { pagefault_enable(); -+ preempt_enable(); ++ preempt_enable_nort(); } #define kmap_atomic_pfn(pfn) kmap_atomic(pfn_to_page(pfn)) @@ -9202,9 +18650,9 @@ diff -Nur linux-4.1.6.orig/include/linux/highmem.h linux-4.1.6/include/linux/hig #endif } -diff -Nur linux-4.1.6.orig/include/linux/hrtimer.h linux-4.1.6/include/linux/hrtimer.h ---- linux-4.1.6.orig/include/linux/hrtimer.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/hrtimer.h 2015-09-08 23:49:06.534041686 +0200 +diff -Nur linux-4.1.10.orig/include/linux/hrtimer.h linux-4.1.10/include/linux/hrtimer.h +--- linux-4.1.10.orig/include/linux/hrtimer.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/hrtimer.h 2015-10-07 18:00:08.000000000 +0200 @@ -111,6 +111,11 @@ enum hrtimer_restart (*function)(struct hrtimer *); struct hrtimer_clock_base *base; @@ -9249,9 +18697,9 @@ diff -Nur linux-4.1.6.orig/include/linux/hrtimer.h linux-4.1.6/include/linux/hrt /* Query timers: */ extern ktime_t hrtimer_get_remaining(const struct hrtimer *timer); extern int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp); -diff -Nur linux-4.1.6.orig/include/linux/idr.h linux-4.1.6/include/linux/idr.h ---- linux-4.1.6.orig/include/linux/idr.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/idr.h 2015-09-08 23:49:06.534041686 +0200 +diff -Nur linux-4.1.10.orig/include/linux/idr.h linux-4.1.10/include/linux/idr.h +--- linux-4.1.10.orig/include/linux/idr.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/idr.h 2015-10-07 18:00:08.000000000 +0200 @@ -95,10 +95,14 @@ * Each idr_preload() should be matched with an invocation of this * function. See idr_preload() for details. @@ -9267,9 +18715,9 @@ diff -Nur linux-4.1.6.orig/include/linux/idr.h linux-4.1.6/include/linux/idr.h /** * idr_find - return pointer for given id -diff -Nur linux-4.1.6.orig/include/linux/init_task.h linux-4.1.6/include/linux/init_task.h ---- linux-4.1.6.orig/include/linux/init_task.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/init_task.h 2015-09-08 23:49:06.542040801 +0200 +diff -Nur linux-4.1.10.orig/include/linux/init_task.h linux-4.1.10/include/linux/init_task.h +--- linux-4.1.10.orig/include/linux/init_task.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/init_task.h 2015-10-07 18:00:08.000000000 +0200 @@ -147,9 +147,16 @@ # define INIT_PERF_EVENTS(tsk) #endif @@ -9296,9 +18744,9 @@ diff -Nur linux-4.1.6.orig/include/linux/init_task.h linux-4.1.6/include/linux/i .pids = { \ [PIDTYPE_PID] = INIT_PID_LINK(PIDTYPE_PID), \ [PIDTYPE_PGID] = INIT_PID_LINK(PIDTYPE_PGID), \ -diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/interrupt.h ---- linux-4.1.6.orig/include/linux/interrupt.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/interrupt.h 2015-09-08 23:49:06.542040801 +0200 +diff -Nur linux-4.1.10.orig/include/linux/interrupt.h linux-4.1.10/include/linux/interrupt.h +--- linux-4.1.10.orig/include/linux/interrupt.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/interrupt.h 2015-10-07 18:00:08.000000000 +0200 @@ -61,6 +61,7 @@ * interrupt handler after suspending interrupts. For system * wakeup devices users need to implement wakeup detection in @@ -9315,7 +18763,23 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i #define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD) -@@ -184,7 +186,7 @@ +@@ -102,6 +104,7 @@ + * @flags: flags (see IRQF_* above) + * @thread_fn: interrupt handler function for threaded interrupts + * @thread: thread pointer for threaded interrupts ++ * @secondary: pointer to secondary irqaction (force threading) + * @thread_flags: flags related to @thread + * @thread_mask: bitmask for keeping track of @thread activity + * @dir: pointer to the proc/irq/NN/name entry +@@ -113,6 +116,7 @@ + struct irqaction *next; + irq_handler_t thread_fn; + struct task_struct *thread; ++ struct irqaction *secondary; + unsigned int irq; + unsigned int flags; + unsigned long thread_flags; +@@ -184,7 +188,7 @@ #ifdef CONFIG_LOCKDEP # define local_irq_enable_in_hardirq() do { } while (0) #else @@ -9324,7 +18788,7 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i #endif extern void disable_irq_nosync(unsigned int irq); -@@ -215,6 +217,7 @@ +@@ -215,6 +219,7 @@ unsigned int irq; struct kref kref; struct work_struct work; @@ -9332,7 +18796,7 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask); void (*release)(struct kref *ref); }; -@@ -377,9 +380,13 @@ +@@ -377,9 +382,13 @@ bool state); #ifdef CONFIG_IRQ_FORCED_THREADING @@ -9347,7 +18811,7 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i #endif #ifndef __ARCH_SET_SOFTIRQ_PENDING -@@ -435,9 +442,10 @@ +@@ -435,9 +444,10 @@ void (*action)(struct softirq_action *); }; @@ -9359,7 +18823,7 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i #ifdef __ARCH_HAS_DO_SOFTIRQ void do_softirq_own_stack(void); #else -@@ -446,6 +454,9 @@ +@@ -446,6 +456,9 @@ __do_softirq(); } #endif @@ -9369,7 +18833,7 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i extern void open_softirq(int nr, void (*action)(struct softirq_action *)); extern void softirq_init(void); -@@ -453,6 +464,7 @@ +@@ -453,6 +466,7 @@ extern void raise_softirq_irqoff(unsigned int nr); extern void raise_softirq(unsigned int nr); @@ -9377,7 +18841,7 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i DECLARE_PER_CPU(struct task_struct *, ksoftirqd); -@@ -474,8 +486,9 @@ +@@ -474,8 +488,9 @@ to be executed on some cpu at least once after this. * If the tasklet is already scheduled, but its execution is still not started, it will be executed only once. @@ -9389,7 +18853,7 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i * Tasklet is strictly serialized wrt itself, but not wrt another tasklets. If client needs some intertask synchronization, he makes it with spinlocks. -@@ -500,27 +513,36 @@ +@@ -500,27 +515,36 @@ enum { TASKLET_STATE_SCHED, /* Tasklet is scheduled for execution */ @@ -9432,7 +18896,7 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i #define tasklet_unlock_wait(t) do { } while (0) #define tasklet_unlock(t) do { } while (0) #endif -@@ -569,12 +591,7 @@ +@@ -569,12 +593,7 @@ smp_mb(); } @@ -9446,7 +18910,7 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i extern void tasklet_kill(struct tasklet_struct *t); extern void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu); extern void tasklet_init(struct tasklet_struct *t, -@@ -605,6 +622,12 @@ +@@ -605,6 +624,12 @@ tasklet_kill(&ttimer->tasklet); } @@ -9459,9 +18923,9 @@ diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/i /* * Autoprobing for irqs: * -diff -Nur linux-4.1.6.orig/include/linux/io-mapping.h linux-4.1.6/include/linux/io-mapping.h ---- linux-4.1.6.orig/include/linux/io-mapping.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/io-mapping.h 2015-09-08 23:49:06.542040801 +0200 +diff -Nur linux-4.1.10.orig/include/linux/io-mapping.h linux-4.1.10/include/linux/io-mapping.h +--- linux-4.1.10.orig/include/linux/io-mapping.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/io-mapping.h 2015-10-07 18:00:08.000000000 +0200 @@ -141,6 +141,7 @@ io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset) @@ -9478,9 +18942,9 @@ diff -Nur linux-4.1.6.orig/include/linux/io-mapping.h linux-4.1.6/include/linux/ } /* Non-atomic map/unmap */ -diff -Nur linux-4.1.6.orig/include/linux/irqdesc.h linux-4.1.6/include/linux/irqdesc.h ---- linux-4.1.6.orig/include/linux/irqdesc.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/irqdesc.h 2015-09-08 23:49:06.546040357 +0200 +diff -Nur linux-4.1.10.orig/include/linux/irqdesc.h linux-4.1.10/include/linux/irqdesc.h +--- linux-4.1.10.orig/include/linux/irqdesc.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irqdesc.h 2015-10-07 18:00:08.000000000 +0200 @@ -63,6 +63,7 @@ unsigned int irqs_unhandled; atomic_t threads_handled; @@ -9489,9 +18953,9 @@ diff -Nur linux-4.1.6.orig/include/linux/irqdesc.h linux-4.1.6/include/linux/irq raw_spinlock_t lock; struct cpumask *percpu_enabled; #ifdef CONFIG_SMP -diff -Nur linux-4.1.6.orig/include/linux/irqflags.h linux-4.1.6/include/linux/irqflags.h ---- linux-4.1.6.orig/include/linux/irqflags.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/irqflags.h 2015-09-08 23:49:06.546040357 +0200 +diff -Nur linux-4.1.10.orig/include/linux/irqflags.h linux-4.1.10/include/linux/irqflags.h +--- linux-4.1.10.orig/include/linux/irqflags.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irqflags.h 2015-10-07 18:00:08.000000000 +0200 @@ -25,8 +25,6 @@ # define trace_softirqs_enabled(p) ((p)->softirqs_enabled) # define trace_hardirq_enter() do { current->hardirq_context++; } while (0) @@ -9542,9 +19006,9 @@ diff -Nur linux-4.1.6.orig/include/linux/irqflags.h linux-4.1.6/include/linux/ir +#endif + #endif -diff -Nur linux-4.1.6.orig/include/linux/irq.h linux-4.1.6/include/linux/irq.h ---- linux-4.1.6.orig/include/linux/irq.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/irq.h 2015-09-08 23:49:06.542040801 +0200 +diff -Nur linux-4.1.10.orig/include/linux/irq.h linux-4.1.10/include/linux/irq.h +--- linux-4.1.10.orig/include/linux/irq.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irq.h 2015-10-07 18:00:08.000000000 +0200 @@ -72,6 +72,7 @@ * IRQ_IS_POLLED - Always polled by another interrupt. Exclude * it from the spurious interrupt detection @@ -9569,9 +19033,9 @@ diff -Nur linux-4.1.6.orig/include/linux/irq.h linux-4.1.6/include/linux/irq.h #define IRQ_NO_BALANCING_MASK (IRQ_PER_CPU | IRQ_NO_BALANCING) -diff -Nur linux-4.1.6.orig/include/linux/irq_work.h linux-4.1.6/include/linux/irq_work.h ---- linux-4.1.6.orig/include/linux/irq_work.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/irq_work.h 2015-09-08 23:49:06.542040801 +0200 +diff -Nur linux-4.1.10.orig/include/linux/irq_work.h linux-4.1.10/include/linux/irq_work.h +--- linux-4.1.10.orig/include/linux/irq_work.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irq_work.h 2015-10-07 18:00:08.000000000 +0200 @@ -16,6 +16,7 @@ #define IRQ_WORK_BUSY 2UL #define IRQ_WORK_FLAGS 3UL @@ -9580,9 +19044,9 @@ diff -Nur linux-4.1.6.orig/include/linux/irq_work.h linux-4.1.6/include/linux/ir struct irq_work { unsigned long flags; -diff -Nur linux-4.1.6.orig/include/linux/jbd_common.h linux-4.1.6/include/linux/jbd_common.h ---- linux-4.1.6.orig/include/linux/jbd_common.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/jbd_common.h 2015-09-08 23:49:06.546040357 +0200 +diff -Nur linux-4.1.10.orig/include/linux/jbd_common.h linux-4.1.10/include/linux/jbd_common.h +--- linux-4.1.10.orig/include/linux/jbd_common.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/jbd_common.h 2015-10-07 18:00:08.000000000 +0200 @@ -15,32 +15,56 @@ static inline void jbd_lock_bh_state(struct buffer_head *bh) @@ -9640,9 +19104,9 @@ diff -Nur linux-4.1.6.orig/include/linux/jbd_common.h linux-4.1.6/include/linux/ } #endif -diff -Nur linux-4.1.6.orig/include/linux/kdb.h linux-4.1.6/include/linux/kdb.h ---- linux-4.1.6.orig/include/linux/kdb.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/kdb.h 2015-09-08 23:49:06.546040357 +0200 +diff -Nur linux-4.1.10.orig/include/linux/kdb.h linux-4.1.10/include/linux/kdb.h +--- linux-4.1.10.orig/include/linux/kdb.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/kdb.h 2015-10-07 18:00:08.000000000 +0200 @@ -167,6 +167,7 @@ extern __printf(1, 2) int kdb_printf(const char *, ...); typedef __printf(1, 2) int (*kdb_printf_t)(const char *, ...); @@ -9659,9 +19123,9 @@ diff -Nur linux-4.1.6.orig/include/linux/kdb.h linux-4.1.6/include/linux/kdb.h static inline void kdb_init(int level) {} static inline int kdb_register(char *cmd, kdb_func_t func, char *usage, char *help, short minlen) { return 0; } -diff -Nur linux-4.1.6.orig/include/linux/kernel.h linux-4.1.6/include/linux/kernel.h ---- linux-4.1.6.orig/include/linux/kernel.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/kernel.h 2015-09-08 23:49:06.546040357 +0200 +diff -Nur linux-4.1.10.orig/include/linux/kernel.h linux-4.1.10/include/linux/kernel.h +--- linux-4.1.10.orig/include/linux/kernel.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/kernel.h 2015-10-07 18:00:08.000000000 +0200 @@ -188,6 +188,9 @@ */ # define might_sleep() \ @@ -9698,9 +19162,9 @@ diff -Nur linux-4.1.6.orig/include/linux/kernel.h linux-4.1.6/include/linux/kern } system_state; #define TAINT_PROPRIETARY_MODULE 0 -diff -Nur linux-4.1.6.orig/include/linux/kvm_host.h linux-4.1.6/include/linux/kvm_host.h ---- linux-4.1.6.orig/include/linux/kvm_host.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/kvm_host.h 2015-09-08 23:49:06.546040357 +0200 +diff -Nur linux-4.1.10.orig/include/linux/kvm_host.h linux-4.1.10/include/linux/kvm_host.h +--- linux-4.1.10.orig/include/linux/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/kvm_host.h 2015-10-07 18:00:08.000000000 +0200 @@ -230,7 +230,7 @@ int fpu_active; @@ -9719,9 +19183,9 @@ diff -Nur linux-4.1.6.orig/include/linux/kvm_host.h linux-4.1.6/include/linux/kv { #ifdef __KVM_HAVE_ARCH_WQP return vcpu->arch.wqp; -diff -Nur linux-4.1.6.orig/include/linux/lglock.h linux-4.1.6/include/linux/lglock.h ---- linux-4.1.6.orig/include/linux/lglock.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/lglock.h 2015-09-08 23:49:06.546040357 +0200 +diff -Nur linux-4.1.10.orig/include/linux/lglock.h linux-4.1.10/include/linux/lglock.h +--- linux-4.1.10.orig/include/linux/lglock.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/lglock.h 2015-10-07 18:00:08.000000000 +0200 @@ -34,22 +34,39 @@ #endif @@ -9777,9 +19241,9 @@ diff -Nur linux-4.1.6.orig/include/linux/lglock.h linux-4.1.6/include/linux/lglo #else /* When !CONFIG_SMP, map lglock to spinlock */ #define lglock spinlock -diff -Nur linux-4.1.6.orig/include/linux/list_bl.h linux-4.1.6/include/linux/list_bl.h ---- linux-4.1.6.orig/include/linux/list_bl.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/list_bl.h 2015-09-08 23:49:06.546040357 +0200 +diff -Nur linux-4.1.10.orig/include/linux/list_bl.h linux-4.1.10/include/linux/list_bl.h +--- linux-4.1.10.orig/include/linux/list_bl.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/list_bl.h 2015-10-07 18:00:08.000000000 +0200 @@ -2,6 +2,7 @@ #define _LINUX_LIST_BL_H @@ -9840,9 +19304,9 @@ diff -Nur linux-4.1.6.orig/include/linux/list_bl.h linux-4.1.6/include/linux/lis } static inline bool hlist_bl_is_locked(struct hlist_bl_head *b) -diff -Nur linux-4.1.6.orig/include/linux/locallock.h linux-4.1.6/include/linux/locallock.h ---- linux-4.1.6.orig/include/linux/locallock.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/locallock.h 2015-09-08 23:49:06.550039912 +0200 +diff -Nur linux-4.1.10.orig/include/linux/locallock.h linux-4.1.10/include/linux/locallock.h +--- linux-4.1.10.orig/include/linux/locallock.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/locallock.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,270 @@ +#ifndef _LINUX_LOCALLOCK_H +#define _LINUX_LOCALLOCK_H @@ -10114,9 +19578,9 @@ diff -Nur linux-4.1.6.orig/include/linux/locallock.h linux-4.1.6/include/linux/l +#endif + +#endif -diff -Nur linux-4.1.6.orig/include/linux/mm_types.h linux-4.1.6/include/linux/mm_types.h ---- linux-4.1.6.orig/include/linux/mm_types.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/mm_types.h 2015-09-08 23:49:06.550039912 +0200 +diff -Nur linux-4.1.10.orig/include/linux/mm_types.h linux-4.1.10/include/linux/mm_types.h +--- linux-4.1.10.orig/include/linux/mm_types.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/mm_types.h 2015-10-07 18:00:08.000000000 +0200 @@ -11,6 +11,7 @@ #include <linux/completion.h> #include <linux/cpumask.h> @@ -10125,7 +19589,7 @@ diff -Nur linux-4.1.6.orig/include/linux/mm_types.h linux-4.1.6/include/linux/mm #include <linux/page-flags-layout.h> #include <asm/page.h> #include <asm/mmu.h> -@@ -462,6 +463,9 @@ +@@ -453,6 +454,9 @@ bool tlb_flush_pending; #endif struct uprobes_state uprobes_state; @@ -10135,9 +19599,552 @@ diff -Nur linux-4.1.6.orig/include/linux/mm_types.h linux-4.1.6/include/linux/mm #ifdef CONFIG_X86_INTEL_MPX /* address of the bounds directory */ void __user *bd_addr; -diff -Nur linux-4.1.6.orig/include/linux/mutex.h linux-4.1.6/include/linux/mutex.h ---- linux-4.1.6.orig/include/linux/mutex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/mutex.h 2015-09-08 23:49:06.774015077 +0200 +diff -Nur linux-4.1.10.orig/include/linux/mm_types.h.orig linux-4.1.10/include/linux/mm_types.h.orig +--- linux-4.1.10.orig/include/linux/mm_types.h.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/mm_types.h.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,539 @@ ++#ifndef _LINUX_MM_TYPES_H ++#define _LINUX_MM_TYPES_H ++ ++#include <linux/auxvec.h> ++#include <linux/types.h> ++#include <linux/threads.h> ++#include <linux/list.h> ++#include <linux/spinlock.h> ++#include <linux/rbtree.h> ++#include <linux/rwsem.h> ++#include <linux/completion.h> ++#include <linux/cpumask.h> ++#include <linux/uprobes.h> ++#include <linux/page-flags-layout.h> ++#include <asm/page.h> ++#include <asm/mmu.h> ++ ++#ifndef AT_VECTOR_SIZE_ARCH ++#define AT_VECTOR_SIZE_ARCH 0 ++#endif ++#define AT_VECTOR_SIZE (2*(AT_VECTOR_SIZE_ARCH + AT_VECTOR_SIZE_BASE + 1)) ++ ++struct address_space; ++struct mem_cgroup; ++ ++#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS) ++#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \ ++ IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) ++#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) ++ ++typedef void compound_page_dtor(struct page *); ++ ++/* ++ * Each physical page in the system has a struct page associated with ++ * it to keep track of whatever it is we are using the page for at the ++ * moment. Note that we have no way to track which tasks are using ++ * a page, though if it is a pagecache page, rmap structures can tell us ++ * who is mapping it. ++ * ++ * The objects in struct page are organized in double word blocks in ++ * order to allows us to use atomic double word operations on portions ++ * of struct page. That is currently only used by slub but the arrangement ++ * allows the use of atomic double word operations on the flags/mapping ++ * and lru list pointers also. ++ */ ++struct page { ++ /* First double word block */ ++ unsigned long flags; /* Atomic flags, some possibly ++ * updated asynchronously */ ++ union { ++ struct address_space *mapping; /* If low bit clear, points to ++ * inode address_space, or NULL. ++ * If page mapped as anonymous ++ * memory, low bit is set, and ++ * it points to anon_vma object: ++ * see PAGE_MAPPING_ANON below. ++ */ ++ void *s_mem; /* slab first object */ ++ }; ++ ++ /* Second double word */ ++ struct { ++ union { ++ pgoff_t index; /* Our offset within mapping. */ ++ void *freelist; /* sl[aou]b first free object */ ++ }; ++ ++ union { ++#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ ++ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) ++ /* Used for cmpxchg_double in slub */ ++ unsigned long counters; ++#else ++ /* ++ * Keep _count separate from slub cmpxchg_double data. ++ * As the rest of the double word is protected by ++ * slab_lock but _count is not. ++ */ ++ unsigned counters; ++#endif ++ ++ struct { ++ ++ union { ++ /* ++ * Count of ptes mapped in ++ * mms, to show when page is ++ * mapped & limit reverse map ++ * searches. ++ * ++ * Used also for tail pages ++ * refcounting instead of ++ * _count. Tail pages cannot ++ * be mapped and keeping the ++ * tail page _count zero at ++ * all times guarantees ++ * get_page_unless_zero() will ++ * never succeed on tail ++ * pages. ++ */ ++ atomic_t _mapcount; ++ ++ struct { /* SLUB */ ++ unsigned inuse:16; ++ unsigned objects:15; ++ unsigned frozen:1; ++ }; ++ int units; /* SLOB */ ++ }; ++ atomic_t _count; /* Usage count, see below. */ ++ }; ++ unsigned int active; /* SLAB */ ++ }; ++ }; ++ ++ /* Third double word block */ ++ union { ++ struct list_head lru; /* Pageout list, eg. active_list ++ * protected by zone->lru_lock ! ++ * Can be used as a generic list ++ * by the page owner. ++ */ ++ struct { /* slub per cpu partial pages */ ++ struct page *next; /* Next partial slab */ ++#ifdef CONFIG_64BIT ++ int pages; /* Nr of partial slabs left */ ++ int pobjects; /* Approximate # of objects */ ++#else ++ short int pages; ++ short int pobjects; ++#endif ++ }; ++ ++ struct slab *slab_page; /* slab fields */ ++ struct rcu_head rcu_head; /* Used by SLAB ++ * when destroying via RCU ++ */ ++ /* First tail page of compound page */ ++ struct { ++ compound_page_dtor *compound_dtor; ++ unsigned long compound_order; ++ }; ++ ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS ++ pgtable_t pmd_huge_pte; /* protected by page->ptl */ ++#endif ++ }; ++ ++ /* Remainder is not double word aligned */ ++ union { ++ unsigned long private; /* Mapping-private opaque data: ++ * usually used for buffer_heads ++ * if PagePrivate set; used for ++ * swp_entry_t if PageSwapCache; ++ * indicates order in the buddy ++ * system if PG_buddy is set. ++ */ ++#if USE_SPLIT_PTE_PTLOCKS ++#if ALLOC_SPLIT_PTLOCKS ++ spinlock_t *ptl; ++#else ++ spinlock_t ptl; ++#endif ++#endif ++ struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */ ++ struct page *first_page; /* Compound tail pages */ ++ }; ++ ++#ifdef CONFIG_MEMCG ++ struct mem_cgroup *mem_cgroup; ++#endif ++ ++ /* ++ * On machines where all RAM is mapped into kernel address space, ++ * we can simply calculate the virtual address. On machines with ++ * highmem some memory is mapped into kernel virtual memory ++ * dynamically, so we need a place to store that address. ++ * Note that this field could be 16 bits on x86 ... ;) ++ * ++ * Architectures with slow multiplication can define ++ * WANT_PAGE_VIRTUAL in asm/page.h ++ */ ++#if defined(WANT_PAGE_VIRTUAL) ++ void *virtual; /* Kernel virtual address (NULL if ++ not kmapped, ie. highmem) */ ++#endif /* WANT_PAGE_VIRTUAL */ ++ ++#ifdef CONFIG_KMEMCHECK ++ /* ++ * kmemcheck wants to track the status of each byte in a page; this ++ * is a pointer to such a status block. NULL if not tracked. ++ */ ++ void *shadow; ++#endif ++ ++#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS ++ int _last_cpupid; ++#endif ++} ++/* ++ * The struct page can be forced to be double word aligned so that atomic ops ++ * on double words work. The SLUB allocator can make use of such a feature. ++ */ ++#ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE ++ __aligned(2 * sizeof(unsigned long)) ++#endif ++; ++ ++struct page_frag { ++ struct page *page; ++#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) ++ __u32 offset; ++ __u32 size; ++#else ++ __u16 offset; ++ __u16 size; ++#endif ++}; ++ ++typedef unsigned long __nocast vm_flags_t; ++ ++/* ++ * A region containing a mapping of a non-memory backed file under NOMMU ++ * conditions. These are held in a global tree and are pinned by the VMAs that ++ * map parts of them. ++ */ ++struct vm_region { ++ struct rb_node vm_rb; /* link in global region tree */ ++ vm_flags_t vm_flags; /* VMA vm_flags */ ++ unsigned long vm_start; /* start address of region */ ++ unsigned long vm_end; /* region initialised to here */ ++ unsigned long vm_top; /* region allocated to here */ ++ unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */ ++ struct file *vm_file; /* the backing file or NULL */ ++ ++ int vm_usage; /* region usage count (access under nommu_region_sem) */ ++ bool vm_icache_flushed : 1; /* true if the icache has been flushed for ++ * this region */ ++}; ++ ++/* ++ * This struct defines a memory VMM memory area. There is one of these ++ * per VM-area/task. A VM area is any part of the process virtual memory ++ * space that has a special rule for the page-fault handlers (ie a shared ++ * library, the executable area etc). ++ */ ++struct vm_area_struct { ++ /* The first cache line has the info for VMA tree walking. */ ++ ++ unsigned long vm_start; /* Our start address within vm_mm. */ ++ unsigned long vm_end; /* The first byte after our end address ++ within vm_mm. */ ++ ++ /* linked list of VM areas per task, sorted by address */ ++ struct vm_area_struct *vm_next, *vm_prev; ++ ++ struct rb_node vm_rb; ++ ++ /* ++ * Largest free memory gap in bytes to the left of this VMA. ++ * Either between this VMA and vma->vm_prev, or between one of the ++ * VMAs below us in the VMA rbtree and its ->vm_prev. This helps ++ * get_unmapped_area find a free area of the right size. ++ */ ++ unsigned long rb_subtree_gap; ++ ++ /* Second cache line starts here. */ ++ ++ struct mm_struct *vm_mm; /* The address space we belong to. */ ++ pgprot_t vm_page_prot; /* Access permissions of this VMA. */ ++ unsigned long vm_flags; /* Flags, see mm.h. */ ++ ++ /* ++ * For areas with an address space and backing store, ++ * linkage into the address_space->i_mmap interval tree. ++ */ ++ struct { ++ struct rb_node rb; ++ unsigned long rb_subtree_last; ++ } shared; ++ ++ /* ++ * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma ++ * list, after a COW of one of the file pages. A MAP_SHARED vma ++ * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack ++ * or brk vma (with NULL file) can only be in an anon_vma list. ++ */ ++ struct list_head anon_vma_chain; /* Serialized by mmap_sem & ++ * page_table_lock */ ++ struct anon_vma *anon_vma; /* Serialized by page_table_lock */ ++ ++ /* Function pointers to deal with this struct. */ ++ const struct vm_operations_struct *vm_ops; ++ ++ /* Information about our backing store: */ ++ unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE ++ units, *not* PAGE_CACHE_SIZE */ ++ struct file * vm_file; /* File we map to (can be NULL). */ ++ void * vm_private_data; /* was vm_pte (shared mem) */ ++ ++#ifndef CONFIG_MMU ++ struct vm_region *vm_region; /* NOMMU mapping region */ ++#endif ++#ifdef CONFIG_NUMA ++ struct mempolicy *vm_policy; /* NUMA policy for the VMA */ ++#endif ++}; ++ ++struct core_thread { ++ struct task_struct *task; ++ struct core_thread *next; ++}; ++ ++struct core_state { ++ atomic_t nr_threads; ++ struct core_thread dumper; ++ struct completion startup; ++}; ++ ++enum { ++ MM_FILEPAGES, ++ MM_ANONPAGES, ++ MM_SWAPENTS, ++ NR_MM_COUNTERS ++}; ++ ++#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU) ++#define SPLIT_RSS_COUNTING ++/* per-thread cached information, */ ++struct task_rss_stat { ++ int events; /* for synchronization threshold */ ++ int count[NR_MM_COUNTERS]; ++}; ++#endif /* USE_SPLIT_PTE_PTLOCKS */ ++ ++struct mm_rss_stat { ++ atomic_long_t count[NR_MM_COUNTERS]; ++}; ++ ++struct kioctx_table; ++struct mm_struct { ++ struct vm_area_struct *mmap; /* list of VMAs */ ++ struct rb_root mm_rb; ++ u32 vmacache_seqnum; /* per-thread vmacache */ ++#ifdef CONFIG_MMU ++ unsigned long (*get_unmapped_area) (struct file *filp, ++ unsigned long addr, unsigned long len, ++ unsigned long pgoff, unsigned long flags); ++#endif ++ unsigned long mmap_base; /* base of mmap area */ ++ unsigned long mmap_legacy_base; /* base of mmap area in bottom-up allocations */ ++ unsigned long task_size; /* size of task vm space */ ++ unsigned long highest_vm_end; /* highest vma end address */ ++ pgd_t * pgd; ++ atomic_t mm_users; /* How many users with user space? */ ++ atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */ ++ atomic_long_t nr_ptes; /* PTE page table pages */ ++#if CONFIG_PGTABLE_LEVELS > 2 ++ atomic_long_t nr_pmds; /* PMD page table pages */ ++#endif ++ int map_count; /* number of VMAs */ ++ ++ spinlock_t page_table_lock; /* Protects page tables and some counters */ ++ struct rw_semaphore mmap_sem; ++ ++ struct list_head mmlist; /* List of maybe swapped mm's. These are globally strung ++ * together off init_mm.mmlist, and are protected ++ * by mmlist_lock ++ */ ++ ++ ++ unsigned long hiwater_rss; /* High-watermark of RSS usage */ ++ unsigned long hiwater_vm; /* High-water virtual memory usage */ ++ ++ unsigned long total_vm; /* Total pages mapped */ ++ unsigned long locked_vm; /* Pages that have PG_mlocked set */ ++ unsigned long pinned_vm; /* Refcount permanently increased */ ++ unsigned long shared_vm; /* Shared pages (files) */ ++ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE */ ++ unsigned long stack_vm; /* VM_GROWSUP/DOWN */ ++ unsigned long def_flags; ++ unsigned long start_code, end_code, start_data, end_data; ++ unsigned long start_brk, brk, start_stack; ++ unsigned long arg_start, arg_end, env_start, env_end; ++ ++ unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */ ++ ++ /* ++ * Special counters, in some configurations protected by the ++ * page_table_lock, in other configurations by being atomic. ++ */ ++ struct mm_rss_stat rss_stat; ++ ++ struct linux_binfmt *binfmt; ++ ++ cpumask_var_t cpu_vm_mask_var; ++ ++ /* Architecture-specific MM context */ ++ mm_context_t context; ++ ++ unsigned long flags; /* Must use atomic bitops to access the bits */ ++ ++ struct core_state *core_state; /* coredumping support */ ++#ifdef CONFIG_AIO ++ spinlock_t ioctx_lock; ++ struct kioctx_table __rcu *ioctx_table; ++#endif ++#ifdef CONFIG_MEMCG ++ /* ++ * "owner" points to a task that is regarded as the canonical ++ * user/owner of this mm. All of the following must be true in ++ * order for it to be changed: ++ * ++ * current == mm->owner ++ * current->mm != mm ++ * new_owner->mm == mm ++ * new_owner->alloc_lock is held ++ */ ++ struct task_struct __rcu *owner; ++#endif ++ ++ /* store ref to file /proc/<pid>/exe symlink points to */ ++ struct file __rcu *exe_file; ++#ifdef CONFIG_MMU_NOTIFIER ++ struct mmu_notifier_mm *mmu_notifier_mm; ++#endif ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS ++ pgtable_t pmd_huge_pte; /* protected by page_table_lock */ ++#endif ++#ifdef CONFIG_CPUMASK_OFFSTACK ++ struct cpumask cpumask_allocation; ++#endif ++#ifdef CONFIG_NUMA_BALANCING ++ /* ++ * numa_next_scan is the next time that the PTEs will be marked ++ * pte_numa. NUMA hinting faults will gather statistics and migrate ++ * pages to new nodes if necessary. ++ */ ++ unsigned long numa_next_scan; ++ ++ /* Restart point for scanning and setting pte_numa */ ++ unsigned long numa_scan_offset; ++ ++ /* numa_scan_seq prevents two threads setting pte_numa */ ++ int numa_scan_seq; ++#endif ++#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) ++ /* ++ * An operation with batched TLB flushing is going on. Anything that ++ * can move process memory needs to flush the TLB when moving a ++ * PROT_NONE or PROT_NUMA mapped page. ++ */ ++ bool tlb_flush_pending; ++#endif ++ struct uprobes_state uprobes_state; ++#ifdef CONFIG_X86_INTEL_MPX ++ /* address of the bounds directory */ ++ void __user *bd_addr; ++#endif ++}; ++ ++static inline void mm_init_cpumask(struct mm_struct *mm) ++{ ++#ifdef CONFIG_CPUMASK_OFFSTACK ++ mm->cpu_vm_mask_var = &mm->cpumask_allocation; ++#endif ++ cpumask_clear(mm->cpu_vm_mask_var); ++} ++ ++/* Future-safe accessor for struct mm_struct's cpu_vm_mask. */ ++static inline cpumask_t *mm_cpumask(struct mm_struct *mm) ++{ ++ return mm->cpu_vm_mask_var; ++} ++ ++#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) ++/* ++ * Memory barriers to keep this state in sync are graciously provided by ++ * the page table locks, outside of which no page table modifications happen. ++ * The barriers below prevent the compiler from re-ordering the instructions ++ * around the memory barriers that are already present in the code. ++ */ ++static inline bool mm_tlb_flush_pending(struct mm_struct *mm) ++{ ++ barrier(); ++ return mm->tlb_flush_pending; ++} ++static inline void set_tlb_flush_pending(struct mm_struct *mm) ++{ ++ mm->tlb_flush_pending = true; ++ ++ /* ++ * Guarantee that the tlb_flush_pending store does not leak into the ++ * critical section updating the page tables ++ */ ++ smp_mb__before_spinlock(); ++} ++/* Clearing is done after a TLB flush, which also provides a barrier. */ ++static inline void clear_tlb_flush_pending(struct mm_struct *mm) ++{ ++ barrier(); ++ mm->tlb_flush_pending = false; ++} ++#else ++static inline bool mm_tlb_flush_pending(struct mm_struct *mm) ++{ ++ return false; ++} ++static inline void set_tlb_flush_pending(struct mm_struct *mm) ++{ ++} ++static inline void clear_tlb_flush_pending(struct mm_struct *mm) ++{ ++} ++#endif ++ ++struct vm_special_mapping ++{ ++ const char *name; ++ struct page **pages; ++}; ++ ++enum tlb_flush_reason { ++ TLB_FLUSH_ON_TASK_SWITCH, ++ TLB_REMOTE_SHOOTDOWN, ++ TLB_LOCAL_SHOOTDOWN, ++ TLB_LOCAL_MM_SHOOTDOWN, ++ NR_TLB_FLUSH_REASONS, ++}; ++ ++ /* ++ * A swap entry has to fit into a "unsigned long", as the entry is hidden ++ * in the "index" field of the swapper address space. ++ */ ++typedef struct { ++ unsigned long val; ++} swp_entry_t; ++ ++#endif /* _LINUX_MM_TYPES_H */ +diff -Nur linux-4.1.10.orig/include/linux/mutex.h linux-4.1.10/include/linux/mutex.h +--- linux-4.1.10.orig/include/linux/mutex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/mutex.h 2015-10-07 18:00:08.000000000 +0200 @@ -19,6 +19,17 @@ #include <asm/processor.h> #include <linux/osq_lock.h> @@ -10179,9 +20186,9 @@ diff -Nur linux-4.1.6.orig/include/linux/mutex.h linux-4.1.6/include/linux/mutex extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); #endif /* __LINUX_MUTEX_H */ -diff -Nur linux-4.1.6.orig/include/linux/mutex_rt.h linux-4.1.6/include/linux/mutex_rt.h ---- linux-4.1.6.orig/include/linux/mutex_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/mutex_rt.h 2015-09-08 23:49:06.774015077 +0200 +diff -Nur linux-4.1.10.orig/include/linux/mutex_rt.h linux-4.1.10/include/linux/mutex_rt.h +--- linux-4.1.10.orig/include/linux/mutex_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/mutex_rt.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,84 @@ +#ifndef __LINUX_MUTEX_RT_H +#define __LINUX_MUTEX_RT_H @@ -10267,9 +20274,9 @@ diff -Nur linux-4.1.6.orig/include/linux/mutex_rt.h linux-4.1.6/include/linux/mu +} while (0) + +#endif -diff -Nur linux-4.1.6.orig/include/linux/netdevice.h linux-4.1.6/include/linux/netdevice.h ---- linux-4.1.6.orig/include/linux/netdevice.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/netdevice.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/netdevice.h linux-4.1.10/include/linux/netdevice.h +--- linux-4.1.10.orig/include/linux/netdevice.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/netdevice.h 2015-10-07 18:00:08.000000000 +0200 @@ -2469,6 +2469,7 @@ unsigned int dropped; struct sk_buff_head input_pkt_queue; @@ -10278,9 +20285,9 @@ diff -Nur linux-4.1.6.orig/include/linux/netdevice.h linux-4.1.6/include/linux/n }; -diff -Nur linux-4.1.6.orig/include/linux/netfilter/x_tables.h linux-4.1.6/include/linux/netfilter/x_tables.h ---- linux-4.1.6.orig/include/linux/netfilter/x_tables.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/netfilter/x_tables.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/netfilter/x_tables.h linux-4.1.10/include/linux/netfilter/x_tables.h +--- linux-4.1.10.orig/include/linux/netfilter/x_tables.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/netfilter/x_tables.h 2015-10-07 18:00:08.000000000 +0200 @@ -3,6 +3,7 @@ @@ -10316,9 +20323,9 @@ diff -Nur linux-4.1.6.orig/include/linux/netfilter/x_tables.h linux-4.1.6/includ } /* -diff -Nur linux-4.1.6.orig/include/linux/notifier.h linux-4.1.6/include/linux/notifier.h ---- linux-4.1.6.orig/include/linux/notifier.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/notifier.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/notifier.h linux-4.1.10/include/linux/notifier.h +--- linux-4.1.10.orig/include/linux/notifier.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/notifier.h 2015-10-07 18:00:08.000000000 +0200 @@ -6,7 +6,7 @@ * * Alan Cox <Alan.Cox@linux.org> @@ -10398,9 +20405,9 @@ diff -Nur linux-4.1.6.orig/include/linux/notifier.h linux-4.1.6/include/linux/no /* CPU notfiers are defined in include/linux/cpu.h. */ /* netdevice notifiers are defined in include/linux/netdevice.h */ -diff -Nur linux-4.1.6.orig/include/linux/percpu.h linux-4.1.6/include/linux/percpu.h ---- linux-4.1.6.orig/include/linux/percpu.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/percpu.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/percpu.h linux-4.1.10/include/linux/percpu.h +--- linux-4.1.10.orig/include/linux/percpu.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/percpu.h 2015-10-07 18:00:08.000000000 +0200 @@ -24,6 +24,35 @@ PERCPU_MODULE_RESERVE) #endif @@ -10437,9 +20444,9 @@ diff -Nur linux-4.1.6.orig/include/linux/percpu.h linux-4.1.6/include/linux/perc /* minimum unit size, also is the maximum supported allocation size */ #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) -diff -Nur linux-4.1.6.orig/include/linux/pid.h linux-4.1.6/include/linux/pid.h ---- linux-4.1.6.orig/include/linux/pid.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/pid.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/pid.h linux-4.1.10/include/linux/pid.h +--- linux-4.1.10.orig/include/linux/pid.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/pid.h 2015-10-07 18:00:08.000000000 +0200 @@ -2,6 +2,7 @@ #define _LINUX_PID_H @@ -10448,9 +20455,9 @@ diff -Nur linux-4.1.6.orig/include/linux/pid.h linux-4.1.6/include/linux/pid.h enum pid_type { -diff -Nur linux-4.1.6.orig/include/linux/preempt.h linux-4.1.6/include/linux/preempt.h ---- linux-4.1.6.orig/include/linux/preempt.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/preempt.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/preempt.h linux-4.1.10/include/linux/preempt.h +--- linux-4.1.10.orig/include/linux/preempt.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/preempt.h 2015-10-07 18:00:08.000000000 +0200 @@ -33,6 +33,20 @@ #define preempt_count_inc() preempt_count_add(1) #define preempt_count_dec() preempt_count_sub(1) @@ -10554,9 +20561,9 @@ diff -Nur linux-4.1.6.orig/include/linux/preempt.h linux-4.1.6/include/linux/pre #ifdef CONFIG_PREEMPT_NOTIFIERS struct preempt_notifier; -diff -Nur linux-4.1.6.orig/include/linux/preempt_mask.h linux-4.1.6/include/linux/preempt_mask.h ---- linux-4.1.6.orig/include/linux/preempt_mask.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/preempt_mask.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/preempt_mask.h linux-4.1.10/include/linux/preempt_mask.h +--- linux-4.1.10.orig/include/linux/preempt_mask.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/preempt_mask.h 2015-10-07 18:00:08.000000000 +0200 @@ -44,16 +44,26 @@ #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) #define NMI_OFFSET (1UL << NMI_SHIFT) @@ -10594,9 +20601,9 @@ diff -Nur linux-4.1.6.orig/include/linux/preempt_mask.h linux-4.1.6/include/linu /* * Are we in NMI context? -diff -Nur linux-4.1.6.orig/include/linux/printk.h linux-4.1.6/include/linux/printk.h ---- linux-4.1.6.orig/include/linux/printk.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/printk.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/printk.h linux-4.1.10/include/linux/printk.h +--- linux-4.1.10.orig/include/linux/printk.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/printk.h 2015-10-07 18:00:08.000000000 +0200 @@ -115,9 +115,11 @@ #ifdef CONFIG_EARLY_PRINTK extern asmlinkage __printf(1, 2) @@ -10609,9 +20616,9 @@ diff -Nur linux-4.1.6.orig/include/linux/printk.h linux-4.1.6/include/linux/prin #endif typedef int(*printk_func_t)(const char *fmt, va_list args); -diff -Nur linux-4.1.6.orig/include/linux/radix-tree.h linux-4.1.6/include/linux/radix-tree.h ---- linux-4.1.6.orig/include/linux/radix-tree.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/radix-tree.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/radix-tree.h linux-4.1.10/include/linux/radix-tree.h +--- linux-4.1.10.orig/include/linux/radix-tree.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/radix-tree.h 2015-10-07 18:00:08.000000000 +0200 @@ -277,8 +277,13 @@ unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root, void ***results, unsigned long *indices, @@ -10635,9 +20642,9 @@ diff -Nur linux-4.1.6.orig/include/linux/radix-tree.h linux-4.1.6/include/linux/ } /** -diff -Nur linux-4.1.6.orig/include/linux/random.h linux-4.1.6/include/linux/random.h ---- linux-4.1.6.orig/include/linux/random.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/random.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/random.h linux-4.1.10/include/linux/random.h +--- linux-4.1.10.orig/include/linux/random.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/random.h 2015-10-07 18:00:08.000000000 +0200 @@ -11,7 +11,7 @@ extern void add_device_randomness(const void *, unsigned int); extern void add_input_randomness(unsigned int type, unsigned int code, @@ -10647,9 +20654,9 @@ diff -Nur linux-4.1.6.orig/include/linux/random.h linux-4.1.6/include/linux/rand extern void get_random_bytes(void *buf, int nbytes); extern void get_random_bytes_arch(void *buf, int nbytes); -diff -Nur linux-4.1.6.orig/include/linux/rcupdate.h linux-4.1.6/include/linux/rcupdate.h ---- linux-4.1.6.orig/include/linux/rcupdate.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rcupdate.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/rcupdate.h linux-4.1.10/include/linux/rcupdate.h +--- linux-4.1.10.orig/include/linux/rcupdate.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rcupdate.h 2015-10-07 18:00:08.000000000 +0200 @@ -167,6 +167,9 @@ #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ @@ -10734,9 +20741,9 @@ diff -Nur linux-4.1.6.orig/include/linux/rcupdate.h linux-4.1.6/include/linux/rc local_bh_enable(); } -diff -Nur linux-4.1.6.orig/include/linux/rcutree.h linux-4.1.6/include/linux/rcutree.h ---- linux-4.1.6.orig/include/linux/rcutree.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rcutree.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/rcutree.h linux-4.1.10/include/linux/rcutree.h +--- linux-4.1.10.orig/include/linux/rcutree.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rcutree.h 2015-10-07 18:00:08.000000000 +0200 @@ -46,7 +46,11 @@ rcu_note_context_switch(); } @@ -10789,9 +20796,9 @@ diff -Nur linux-4.1.6.orig/include/linux/rcutree.h linux-4.1.6/include/linux/rcu void rcu_all_qs(void); #endif /* __LINUX_RCUTREE_H */ -diff -Nur linux-4.1.6.orig/include/linux/rtmutex.h linux-4.1.6/include/linux/rtmutex.h ---- linux-4.1.6.orig/include/linux/rtmutex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rtmutex.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/rtmutex.h linux-4.1.10/include/linux/rtmutex.h +--- linux-4.1.10.orig/include/linux/rtmutex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rtmutex.h 2015-10-07 18:00:08.000000000 +0200 @@ -14,10 +14,14 @@ #include <linux/linkage.h> @@ -10865,9 +20872,9 @@ diff -Nur linux-4.1.6.orig/include/linux/rtmutex.h linux-4.1.6/include/linux/rtm extern int rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout); -diff -Nur linux-4.1.6.orig/include/linux/rwlock_rt.h linux-4.1.6/include/linux/rwlock_rt.h ---- linux-4.1.6.orig/include/linux/rwlock_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/rwlock_rt.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/rwlock_rt.h linux-4.1.10/include/linux/rwlock_rt.h +--- linux-4.1.10.orig/include/linux/rwlock_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/rwlock_rt.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,99 @@ +#ifndef __LINUX_RWLOCK_RT_H +#define __LINUX_RWLOCK_RT_H @@ -10968,9 +20975,9 @@ diff -Nur linux-4.1.6.orig/include/linux/rwlock_rt.h linux-4.1.6/include/linux/r + } while (0) + +#endif -diff -Nur linux-4.1.6.orig/include/linux/rwlock_types.h linux-4.1.6/include/linux/rwlock_types.h ---- linux-4.1.6.orig/include/linux/rwlock_types.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rwlock_types.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/rwlock_types.h linux-4.1.10/include/linux/rwlock_types.h +--- linux-4.1.10.orig/include/linux/rwlock_types.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rwlock_types.h 2015-10-07 18:00:08.000000000 +0200 @@ -1,6 +1,10 @@ #ifndef __LINUX_RWLOCK_TYPES_H #define __LINUX_RWLOCK_TYPES_H @@ -10991,9 +20998,9 @@ diff -Nur linux-4.1.6.orig/include/linux/rwlock_types.h linux-4.1.6/include/linu + rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) #endif /* __LINUX_RWLOCK_TYPES_H */ -diff -Nur linux-4.1.6.orig/include/linux/rwlock_types_rt.h linux-4.1.6/include/linux/rwlock_types_rt.h ---- linux-4.1.6.orig/include/linux/rwlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/rwlock_types_rt.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/rwlock_types_rt.h linux-4.1.10/include/linux/rwlock_types_rt.h +--- linux-4.1.10.orig/include/linux/rwlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/rwlock_types_rt.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,33 @@ +#ifndef __LINUX_RWLOCK_TYPES_RT_H +#define __LINUX_RWLOCK_TYPES_RT_H @@ -11028,9 +21035,9 @@ diff -Nur linux-4.1.6.orig/include/linux/rwlock_types_rt.h linux-4.1.6/include/l + rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) + +#endif -diff -Nur linux-4.1.6.orig/include/linux/rwsem.h linux-4.1.6/include/linux/rwsem.h ---- linux-4.1.6.orig/include/linux/rwsem.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rwsem.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/rwsem.h linux-4.1.10/include/linux/rwsem.h +--- linux-4.1.10.orig/include/linux/rwsem.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rwsem.h 2015-10-07 18:00:08.000000000 +0200 @@ -18,6 +18,10 @@ #include <linux/osq_lock.h> #endif @@ -11049,9 +21056,9 @@ diff -Nur linux-4.1.6.orig/include/linux/rwsem.h linux-4.1.6/include/linux/rwsem +#endif /* !PREEMPT_RT_FULL */ + #endif /* _LINUX_RWSEM_H */ -diff -Nur linux-4.1.6.orig/include/linux/rwsem_rt.h linux-4.1.6/include/linux/rwsem_rt.h ---- linux-4.1.6.orig/include/linux/rwsem_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/rwsem_rt.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/rwsem_rt.h linux-4.1.10/include/linux/rwsem_rt.h +--- linux-4.1.10.orig/include/linux/rwsem_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/rwsem_rt.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,140 @@ +#ifndef _LINUX_RWSEM_RT_H +#define _LINUX_RWSEM_RT_H @@ -11193,9 +21200,9 @@ diff -Nur linux-4.1.6.orig/include/linux/rwsem_rt.h linux-4.1.6/include/linux/rw +} +#endif +#endif -diff -Nur linux-4.1.6.orig/include/linux/sched.h linux-4.1.6/include/linux/sched.h ---- linux-4.1.6.orig/include/linux/sched.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/sched.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/sched.h linux-4.1.10/include/linux/sched.h +--- linux-4.1.10.orig/include/linux/sched.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/sched.h 2015-10-07 18:00:08.000000000 +0200 @@ -26,6 +26,7 @@ #include <linux/nodemask.h> #include <linux/mm_types.h> @@ -11619,9 +21626,9 @@ diff -Nur linux-4.1.6.orig/include/linux/sched.h linux-4.1.6/include/linux/sched extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); extern long sched_getaffinity(pid_t pid, struct cpumask *mask); -diff -Nur linux-4.1.6.orig/include/linux/seqlock.h linux-4.1.6/include/linux/seqlock.h ---- linux-4.1.6.orig/include/linux/seqlock.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/seqlock.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/seqlock.h linux-4.1.10/include/linux/seqlock.h +--- linux-4.1.10.orig/include/linux/seqlock.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/seqlock.h 2015-10-07 18:00:08.000000000 +0200 @@ -219,20 +219,30 @@ return __read_seqcount_retry(s, start); } @@ -11751,9 +21758,9 @@ diff -Nur linux-4.1.6.orig/include/linux/seqlock.h linux-4.1.6/include/linux/seq spin_unlock_irqrestore(&sl->lock, flags); } -diff -Nur linux-4.1.6.orig/include/linux/signal.h linux-4.1.6/include/linux/signal.h ---- linux-4.1.6.orig/include/linux/signal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/signal.h 2015-09-08 23:49:06.778014632 +0200 +diff -Nur linux-4.1.10.orig/include/linux/signal.h linux-4.1.10/include/linux/signal.h +--- linux-4.1.10.orig/include/linux/signal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/signal.h 2015-10-07 18:00:08.000000000 +0200 @@ -218,6 +218,7 @@ } @@ -11762,9 +21769,9 @@ diff -Nur linux-4.1.6.orig/include/linux/signal.h linux-4.1.6/include/linux/sign /* Test if 'sig' is valid signal. Use this instead of testing _NSIG directly */ static inline int valid_signal(unsigned long sig) -diff -Nur linux-4.1.6.orig/include/linux/skbuff.h linux-4.1.6/include/linux/skbuff.h ---- linux-4.1.6.orig/include/linux/skbuff.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/skbuff.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/skbuff.h linux-4.1.10/include/linux/skbuff.h +--- linux-4.1.10.orig/include/linux/skbuff.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/skbuff.h 2015-10-07 18:00:08.000000000 +0200 @@ -187,6 +187,7 @@ __u32 qlen; @@ -11786,9 +21793,9 @@ diff -Nur linux-4.1.6.orig/include/linux/skbuff.h linux-4.1.6/include/linux/skbu static inline void skb_queue_head_init_class(struct sk_buff_head *list, struct lock_class_key *class) { -diff -Nur linux-4.1.6.orig/include/linux/smp.h linux-4.1.6/include/linux/smp.h ---- linux-4.1.6.orig/include/linux/smp.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/smp.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/smp.h linux-4.1.10/include/linux/smp.h +--- linux-4.1.10.orig/include/linux/smp.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/smp.h 2015-10-07 18:00:08.000000000 +0200 @@ -185,6 +185,9 @@ #define get_cpu() ({ preempt_disable(); smp_processor_id(); }) #define put_cpu() preempt_enable() @@ -11799,9 +21806,9 @@ diff -Nur linux-4.1.6.orig/include/linux/smp.h linux-4.1.6/include/linux/smp.h /* * Callback to arch code if there's nosmp or maxcpus=0 on the * boot command line: -diff -Nur linux-4.1.6.orig/include/linux/spinlock_api_smp.h linux-4.1.6/include/linux/spinlock_api_smp.h ---- linux-4.1.6.orig/include/linux/spinlock_api_smp.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/spinlock_api_smp.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/spinlock_api_smp.h linux-4.1.10/include/linux/spinlock_api_smp.h +--- linux-4.1.10.orig/include/linux/spinlock_api_smp.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/spinlock_api_smp.h 2015-10-07 18:00:08.000000000 +0200 @@ -189,6 +189,8 @@ return 0; } @@ -11812,9 +21819,9 @@ diff -Nur linux-4.1.6.orig/include/linux/spinlock_api_smp.h linux-4.1.6/include/ +#endif #endif /* __LINUX_SPINLOCK_API_SMP_H */ -diff -Nur linux-4.1.6.orig/include/linux/spinlock.h linux-4.1.6/include/linux/spinlock.h ---- linux-4.1.6.orig/include/linux/spinlock.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/spinlock.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/spinlock.h linux-4.1.10/include/linux/spinlock.h +--- linux-4.1.10.orig/include/linux/spinlock.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/spinlock.h 2015-10-07 18:00:08.000000000 +0200 @@ -281,7 +281,11 @@ #define raw_spin_can_lock(lock) (!raw_spin_is_locked(lock)) @@ -11846,9 +21853,9 @@ diff -Nur linux-4.1.6.orig/include/linux/spinlock.h linux-4.1.6/include/linux/sp +#endif /* !PREEMPT_RT_FULL */ + #endif /* __LINUX_SPINLOCK_H */ -diff -Nur linux-4.1.6.orig/include/linux/spinlock_rt.h linux-4.1.6/include/linux/spinlock_rt.h ---- linux-4.1.6.orig/include/linux/spinlock_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/spinlock_rt.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/spinlock_rt.h linux-4.1.10/include/linux/spinlock_rt.h +--- linux-4.1.10.orig/include/linux/spinlock_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_rt.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,174 @@ +#ifndef __LINUX_SPINLOCK_RT_H +#define __LINUX_SPINLOCK_RT_H @@ -12024,9 +22031,9 @@ diff -Nur linux-4.1.6.orig/include/linux/spinlock_rt.h linux-4.1.6/include/linux + atomic_dec_and_spin_lock(atomic, lock) + +#endif -diff -Nur linux-4.1.6.orig/include/linux/spinlock_types.h linux-4.1.6/include/linux/spinlock_types.h ---- linux-4.1.6.orig/include/linux/spinlock_types.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/spinlock_types.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types.h linux-4.1.10/include/linux/spinlock_types.h +--- linux-4.1.10.orig/include/linux/spinlock_types.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/spinlock_types.h 2015-10-07 18:00:08.000000000 +0200 @@ -9,80 +9,15 @@ * Released under the General Public License (GPL). */ @@ -12115,9 +22122,9 @@ diff -Nur linux-4.1.6.orig/include/linux/spinlock_types.h linux-4.1.6/include/li -#include <linux/rwlock_types.h> - #endif /* __LINUX_SPINLOCK_TYPES_H */ -diff -Nur linux-4.1.6.orig/include/linux/spinlock_types_nort.h linux-4.1.6/include/linux/spinlock_types_nort.h ---- linux-4.1.6.orig/include/linux/spinlock_types_nort.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/spinlock_types_nort.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_nort.h linux-4.1.10/include/linux/spinlock_types_nort.h +--- linux-4.1.10.orig/include/linux/spinlock_types_nort.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_types_nort.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,33 @@ +#ifndef __LINUX_SPINLOCK_TYPES_NORT_H +#define __LINUX_SPINLOCK_TYPES_NORT_H @@ -12152,9 +22159,9 @@ diff -Nur linux-4.1.6.orig/include/linux/spinlock_types_nort.h linux-4.1.6/inclu +#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) + +#endif -diff -Nur linux-4.1.6.orig/include/linux/spinlock_types_raw.h linux-4.1.6/include/linux/spinlock_types_raw.h ---- linux-4.1.6.orig/include/linux/spinlock_types_raw.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/spinlock_types_raw.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_raw.h linux-4.1.10/include/linux/spinlock_types_raw.h +--- linux-4.1.10.orig/include/linux/spinlock_types_raw.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_types_raw.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,56 @@ +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H +#define __LINUX_SPINLOCK_TYPES_RAW_H @@ -12212,9 +22219,9 @@ diff -Nur linux-4.1.6.orig/include/linux/spinlock_types_raw.h linux-4.1.6/includ +#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x) + +#endif -diff -Nur linux-4.1.6.orig/include/linux/spinlock_types_rt.h linux-4.1.6/include/linux/spinlock_types_rt.h ---- linux-4.1.6.orig/include/linux/spinlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/spinlock_types_rt.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_rt.h linux-4.1.10/include/linux/spinlock_types_rt.h +--- linux-4.1.10.orig/include/linux/spinlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_types_rt.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,51 @@ +#ifndef __LINUX_SPINLOCK_TYPES_RT_H +#define __LINUX_SPINLOCK_TYPES_RT_H @@ -12267,9 +22274,9 @@ diff -Nur linux-4.1.6.orig/include/linux/spinlock_types_rt.h linux-4.1.6/include + spinlock_t name __cacheline_aligned_in_smp = __SPIN_LOCK_UNLOCKED(name) + +#endif -diff -Nur linux-4.1.6.orig/include/linux/srcu.h linux-4.1.6/include/linux/srcu.h ---- linux-4.1.6.orig/include/linux/srcu.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/srcu.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/srcu.h linux-4.1.10/include/linux/srcu.h +--- linux-4.1.10.orig/include/linux/srcu.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/srcu.h 2015-10-07 18:00:08.000000000 +0200 @@ -84,10 +84,10 @@ void process_srcu(struct work_struct *work); @@ -12292,9 +22299,9 @@ diff -Nur linux-4.1.6.orig/include/linux/srcu.h linux-4.1.6/include/linux/srcu.h #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */) #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static) -diff -Nur linux-4.1.6.orig/include/linux/swap.h linux-4.1.6/include/linux/swap.h ---- linux-4.1.6.orig/include/linux/swap.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/swap.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/swap.h linux-4.1.10/include/linux/swap.h +--- linux-4.1.10.orig/include/linux/swap.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/swap.h 2015-10-07 18:00:08.000000000 +0200 @@ -11,6 +11,7 @@ #include <linux/fs.h> #include <linux/atomic.h> @@ -12321,9 +22328,9 @@ diff -Nur linux-4.1.6.orig/include/linux/swap.h linux-4.1.6/include/linux/swap.h extern void lru_cache_add(struct page *); extern void lru_cache_add_anon(struct page *page); extern void lru_cache_add_file(struct page *page); -diff -Nur linux-4.1.6.orig/include/linux/thread_info.h linux-4.1.6/include/linux/thread_info.h ---- linux-4.1.6.orig/include/linux/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/thread_info.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/thread_info.h linux-4.1.10/include/linux/thread_info.h +--- linux-4.1.10.orig/include/linux/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/thread_info.h 2015-10-07 18:00:08.000000000 +0200 @@ -102,7 +102,17 @@ #define test_thread_flag(flag) \ test_ti_thread_flag(current_thread_info(), flag) @@ -12343,9 +22350,9 @@ diff -Nur linux-4.1.6.orig/include/linux/thread_info.h linux-4.1.6/include/linux #if defined TIF_RESTORE_SIGMASK && !defined HAVE_SET_RESTORE_SIGMASK /* -diff -Nur linux-4.1.6.orig/include/linux/timer.h linux-4.1.6/include/linux/timer.h ---- linux-4.1.6.orig/include/linux/timer.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/timer.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/timer.h linux-4.1.10/include/linux/timer.h +--- linux-4.1.10.orig/include/linux/timer.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/timer.h 2015-10-07 18:00:08.000000000 +0200 @@ -241,7 +241,7 @@ extern int try_to_del_timer_sync(struct timer_list *timer); @@ -12355,9 +22362,9 @@ diff -Nur linux-4.1.6.orig/include/linux/timer.h linux-4.1.6/include/linux/timer extern int del_timer_sync(struct timer_list *timer); #else # define del_timer_sync(t) del_timer(t) -diff -Nur linux-4.1.6.orig/include/linux/uaccess.h linux-4.1.6/include/linux/uaccess.h ---- linux-4.1.6.orig/include/linux/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/uaccess.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/uaccess.h linux-4.1.10/include/linux/uaccess.h +--- linux-4.1.10.orig/include/linux/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/uaccess.h 2015-10-07 18:00:08.000000000 +0200 @@ -1,21 +1,31 @@ #ifndef __LINUX_UACCESS_H__ #define __LINUX_UACCESS_H__ @@ -12436,9 +22443,9 @@ diff -Nur linux-4.1.6.orig/include/linux/uaccess.h linux-4.1.6/include/linux/uac #ifndef ARCH_HAS_NOCACHE_UACCESS static inline unsigned long __copy_from_user_inatomic_nocache(void *to, -diff -Nur linux-4.1.6.orig/include/linux/uprobes.h linux-4.1.6/include/linux/uprobes.h ---- linux-4.1.6.orig/include/linux/uprobes.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/uprobes.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/uprobes.h linux-4.1.10/include/linux/uprobes.h +--- linux-4.1.10.orig/include/linux/uprobes.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/uprobes.h 2015-10-07 18:00:08.000000000 +0200 @@ -27,6 +27,7 @@ #include <linux/errno.h> #include <linux/rbtree.h> @@ -12447,9 +22454,9 @@ diff -Nur linux-4.1.6.orig/include/linux/uprobes.h linux-4.1.6/include/linux/upr struct vm_area_struct; struct mm_struct; -diff -Nur linux-4.1.6.orig/include/linux/vmstat.h linux-4.1.6/include/linux/vmstat.h ---- linux-4.1.6.orig/include/linux/vmstat.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/vmstat.h 2015-09-08 23:49:06.782014188 +0200 +diff -Nur linux-4.1.10.orig/include/linux/vmstat.h linux-4.1.10/include/linux/vmstat.h +--- linux-4.1.10.orig/include/linux/vmstat.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/vmstat.h 2015-10-07 18:00:08.000000000 +0200 @@ -33,7 +33,9 @@ */ static inline void __count_vm_event(enum vm_event_item item) @@ -12470,9 +22477,9 @@ diff -Nur linux-4.1.6.orig/include/linux/vmstat.h linux-4.1.6/include/linux/vmst } static inline void count_vm_events(enum vm_event_item item, long delta) -diff -Nur linux-4.1.6.orig/include/linux/wait.h linux-4.1.6/include/linux/wait.h ---- linux-4.1.6.orig/include/linux/wait.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/wait.h 2015-09-08 23:49:06.786013744 +0200 +diff -Nur linux-4.1.10.orig/include/linux/wait.h linux-4.1.10/include/linux/wait.h +--- linux-4.1.10.orig/include/linux/wait.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/wait.h 2015-10-07 18:00:08.000000000 +0200 @@ -8,6 +8,7 @@ #include <linux/spinlock.h> #include <asm/current.h> @@ -12481,9 +22488,9 @@ diff -Nur linux-4.1.6.orig/include/linux/wait.h linux-4.1.6/include/linux/wait.h typedef struct __wait_queue wait_queue_t; typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key); -diff -Nur linux-4.1.6.orig/include/linux/wait-simple.h linux-4.1.6/include/linux/wait-simple.h ---- linux-4.1.6.orig/include/linux/wait-simple.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/wait-simple.h 2015-09-08 23:49:06.786013744 +0200 +diff -Nur linux-4.1.10.orig/include/linux/wait-simple.h linux-4.1.10/include/linux/wait-simple.h +--- linux-4.1.10.orig/include/linux/wait-simple.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/wait-simple.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,207 @@ +#ifndef _LINUX_WAIT_SIMPLE_H +#define _LINUX_WAIT_SIMPLE_H @@ -12692,9 +22699,9 @@ diff -Nur linux-4.1.6.orig/include/linux/wait-simple.h linux-4.1.6/include/linux +}) + +#endif -diff -Nur linux-4.1.6.orig/include/linux/work-simple.h linux-4.1.6/include/linux/work-simple.h ---- linux-4.1.6.orig/include/linux/work-simple.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/work-simple.h 2015-09-08 23:49:06.786013744 +0200 +diff -Nur linux-4.1.10.orig/include/linux/work-simple.h linux-4.1.10/include/linux/work-simple.h +--- linux-4.1.10.orig/include/linux/work-simple.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/work-simple.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,24 @@ +#ifndef _LINUX_SWORK_H +#define _LINUX_SWORK_H @@ -12720,9 +22727,9 @@ diff -Nur linux-4.1.6.orig/include/linux/work-simple.h linux-4.1.6/include/linux +void swork_put(void); + +#endif /* _LINUX_SWORK_H */ -diff -Nur linux-4.1.6.orig/include/net/dst.h linux-4.1.6/include/net/dst.h ---- linux-4.1.6.orig/include/net/dst.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/net/dst.h 2015-09-08 23:49:06.786013744 +0200 +diff -Nur linux-4.1.10.orig/include/net/dst.h linux-4.1.10/include/net/dst.h +--- linux-4.1.10.orig/include/net/dst.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/net/dst.h 2015-10-07 18:00:08.000000000 +0200 @@ -403,7 +403,7 @@ static inline int dst_neigh_output(struct dst_entry *dst, struct neighbour *n, struct sk_buff *skb) @@ -12732,9 +22739,9 @@ diff -Nur linux-4.1.6.orig/include/net/dst.h linux-4.1.6/include/net/dst.h if (dst->pending_confirm) { unsigned long now = jiffies; -diff -Nur linux-4.1.6.orig/include/net/neighbour.h linux-4.1.6/include/net/neighbour.h ---- linux-4.1.6.orig/include/net/neighbour.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/net/neighbour.h 2015-09-08 23:49:07.409944556 +0200 +diff -Nur linux-4.1.10.orig/include/net/neighbour.h linux-4.1.10/include/net/neighbour.h +--- linux-4.1.10.orig/include/net/neighbour.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/net/neighbour.h 2015-10-07 18:00:08.000000000 +0200 @@ -445,7 +445,7 @@ } #endif @@ -12753,9 +22760,9 @@ diff -Nur linux-4.1.6.orig/include/net/neighbour.h linux-4.1.6/include/net/neigh const struct net_device *dev) { unsigned int seq; -diff -Nur linux-4.1.6.orig/include/net/netns/ipv4.h linux-4.1.6/include/net/netns/ipv4.h ---- linux-4.1.6.orig/include/net/netns/ipv4.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/net/netns/ipv4.h 2015-09-08 23:49:08.077870489 +0200 +diff -Nur linux-4.1.10.orig/include/net/netns/ipv4.h linux-4.1.10/include/net/netns/ipv4.h +--- linux-4.1.10.orig/include/net/netns/ipv4.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/net/netns/ipv4.h 2015-10-07 18:00:08.000000000 +0200 @@ -69,6 +69,7 @@ int sysctl_icmp_echo_ignore_all; @@ -12764,9 +22771,9 @@ diff -Nur linux-4.1.6.orig/include/net/netns/ipv4.h linux-4.1.6/include/net/netn int sysctl_icmp_ignore_bogus_error_responses; int sysctl_icmp_ratelimit; int sysctl_icmp_ratemask; -diff -Nur linux-4.1.6.orig/include/trace/events/hist.h linux-4.1.6/include/trace/events/hist.h ---- linux-4.1.6.orig/include/trace/events/hist.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/trace/events/hist.h 2015-09-08 23:49:08.077870489 +0200 +diff -Nur linux-4.1.10.orig/include/trace/events/hist.h linux-4.1.10/include/trace/events/hist.h +--- linux-4.1.10.orig/include/trace/events/hist.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/trace/events/hist.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,72 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM hist @@ -12840,9 +22847,9 @@ diff -Nur linux-4.1.6.orig/include/trace/events/hist.h linux-4.1.6/include/trace + +/* This part must be outside protection */ +#include <trace/define_trace.h> -diff -Nur linux-4.1.6.orig/include/trace/events/latency_hist.h linux-4.1.6/include/trace/events/latency_hist.h ---- linux-4.1.6.orig/include/trace/events/latency_hist.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/trace/events/latency_hist.h 2015-09-08 23:49:08.077870489 +0200 +diff -Nur linux-4.1.10.orig/include/trace/events/latency_hist.h linux-4.1.10/include/trace/events/latency_hist.h +--- linux-4.1.10.orig/include/trace/events/latency_hist.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/trace/events/latency_hist.h 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,29 @@ +#ifndef _LATENCY_HIST_H +#define _LATENCY_HIST_H @@ -12873,9 +22880,9 @@ diff -Nur linux-4.1.6.orig/include/trace/events/latency_hist.h linux-4.1.6/inclu +} + +#endif /* _LATENCY_HIST_H */ -diff -Nur linux-4.1.6.orig/init/Kconfig linux-4.1.6/init/Kconfig ---- linux-4.1.6.orig/init/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/init/Kconfig 2015-09-08 23:49:08.077870489 +0200 +diff -Nur linux-4.1.10.orig/init/Kconfig linux-4.1.10/init/Kconfig +--- linux-4.1.10.orig/init/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/init/Kconfig 2015-10-07 18:00:08.000000000 +0200 @@ -637,7 +637,7 @@ config RCU_FAST_NO_HZ @@ -12927,9 +22934,9 @@ diff -Nur linux-4.1.6.orig/init/Kconfig linux-4.1.6/init/Kconfig bool "SLUB per cpu partial cache" help Per cpu partial caches accellerate objects allocation and freeing -diff -Nur linux-4.1.6.orig/init/main.c linux-4.1.6/init/main.c ---- linux-4.1.6.orig/init/main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/init/main.c 2015-09-08 23:49:08.081870045 +0200 +diff -Nur linux-4.1.10.orig/init/main.c linux-4.1.10/init/main.c +--- linux-4.1.10.orig/init/main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/init/main.c 2015-10-07 18:00:08.000000000 +0200 @@ -525,6 +525,7 @@ setup_command_line(command_line); setup_nr_cpu_ids(); @@ -12938,18 +22945,18 @@ diff -Nur linux-4.1.6.orig/init/main.c linux-4.1.6/init/main.c smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ build_all_zonelists(NULL, NULL); -diff -Nur linux-4.1.6.orig/init/Makefile linux-4.1.6/init/Makefile ---- linux-4.1.6.orig/init/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/init/Makefile 2015-09-08 23:49:08.077870489 +0200 +diff -Nur linux-4.1.10.orig/init/Makefile linux-4.1.10/init/Makefile +--- linux-4.1.10.orig/init/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/init/Makefile 2015-10-07 18:00:08.000000000 +0200 @@ -33,4 +33,4 @@ include/generated/compile.h: FORCE @$($(quiet)chk_compile.h) $(Q)$(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@ \ - "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CC) $(KBUILD_CFLAGS)" + "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CONFIG_PREEMPT_RT_FULL)" "$(CC) $(KBUILD_CFLAGS)" -diff -Nur linux-4.1.6.orig/ipc/mqueue.c linux-4.1.6/ipc/mqueue.c ---- linux-4.1.6.orig/ipc/mqueue.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/ipc/mqueue.c 2015-09-08 23:49:08.081870045 +0200 +diff -Nur linux-4.1.10.orig/ipc/mqueue.c linux-4.1.10/ipc/mqueue.c +--- linux-4.1.10.orig/ipc/mqueue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/ipc/mqueue.c 2015-10-07 18:00:08.000000000 +0200 @@ -47,8 +47,7 @@ #define RECV 1 @@ -13090,9 +23097,9 @@ diff -Nur linux-4.1.6.orig/ipc/mqueue.c linux-4.1.6/ipc/mqueue.c ret = 0; } if (ret == 0) { -diff -Nur linux-4.1.6.orig/ipc/msg.c linux-4.1.6/ipc/msg.c ---- linux-4.1.6.orig/ipc/msg.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/ipc/msg.c 2015-09-08 23:49:08.081870045 +0200 +diff -Nur linux-4.1.10.orig/ipc/msg.c linux-4.1.10/ipc/msg.c +--- linux-4.1.10.orig/ipc/msg.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/ipc/msg.c 2015-10-07 18:00:08.000000000 +0200 @@ -188,6 +188,12 @@ struct msg_receiver *msr, *t; @@ -13142,10 +23149,10 @@ diff -Nur linux-4.1.6.orig/ipc/msg.c linux-4.1.6/ipc/msg.c return 0; } -diff -Nur linux-4.1.6.orig/ipc/sem.c linux-4.1.6/ipc/sem.c ---- linux-4.1.6.orig/ipc/sem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/ipc/sem.c 2015-09-08 23:49:08.085869602 +0200 -@@ -680,6 +680,13 @@ +diff -Nur linux-4.1.10.orig/ipc/sem.c linux-4.1.10/ipc/sem.c +--- linux-4.1.10.orig/ipc/sem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/ipc/sem.c 2015-10-07 18:00:08.000000000 +0200 +@@ -690,6 +690,13 @@ static void wake_up_sem_queue_prepare(struct list_head *pt, struct sem_queue *q, int error) { @@ -13159,7 +23166,7 @@ diff -Nur linux-4.1.6.orig/ipc/sem.c linux-4.1.6/ipc/sem.c if (list_empty(pt)) { /* * Hold preempt off so that we don't get preempted and have the -@@ -691,6 +698,7 @@ +@@ -701,6 +708,7 @@ q->pid = error; list_add_tail(&q->list, pt); @@ -13167,7 +23174,7 @@ diff -Nur linux-4.1.6.orig/ipc/sem.c linux-4.1.6/ipc/sem.c } /** -@@ -704,6 +712,7 @@ +@@ -714,6 +722,7 @@ */ static void wake_up_sem_queue_do(struct list_head *pt) { @@ -13175,7 +23182,7 @@ diff -Nur linux-4.1.6.orig/ipc/sem.c linux-4.1.6/ipc/sem.c struct sem_queue *q, *t; int did_something; -@@ -716,6 +725,7 @@ +@@ -726,6 +735,7 @@ } if (did_something) preempt_enable(); @@ -13183,10 +23190,10 @@ diff -Nur linux-4.1.6.orig/ipc/sem.c linux-4.1.6/ipc/sem.c } static void unlink_queue(struct sem_array *sma, struct sem_queue *q) -diff -Nur linux-4.1.6.orig/kernel/cgroup.c linux-4.1.6/kernel/cgroup.c ---- linux-4.1.6.orig/kernel/cgroup.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/cgroup.c 2015-09-08 23:49:08.089869160 +0200 -@@ -4421,10 +4421,10 @@ +diff -Nur linux-4.1.10.orig/kernel/cgroup.c linux-4.1.10/kernel/cgroup.c +--- linux-4.1.10.orig/kernel/cgroup.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/cgroup.c 2015-10-07 18:00:08.000000000 +0200 +@@ -4422,10 +4422,10 @@ queue_work(cgroup_destroy_wq, &css->destroy_work); } @@ -13199,7 +23206,7 @@ diff -Nur linux-4.1.6.orig/kernel/cgroup.c linux-4.1.6/kernel/cgroup.c struct cgroup_subsys *ss = css->ss; struct cgroup *cgrp = css->cgroup; -@@ -4463,8 +4463,8 @@ +@@ -4464,8 +4464,8 @@ struct cgroup_subsys_state *css = container_of(ref, struct cgroup_subsys_state, refcnt); @@ -13210,7 +23217,7 @@ diff -Nur linux-4.1.6.orig/kernel/cgroup.c linux-4.1.6/kernel/cgroup.c } static void init_and_link_css(struct cgroup_subsys_state *css, -@@ -5068,6 +5068,7 @@ +@@ -5069,6 +5069,7 @@ */ cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); BUG_ON(!cgroup_destroy_wq); @@ -13218,9 +23225,5615 @@ diff -Nur linux-4.1.6.orig/kernel/cgroup.c linux-4.1.6/kernel/cgroup.c /* * Used to destroy pidlists and separate to serve as flush domain. -diff -Nur linux-4.1.6.orig/kernel/cpu.c linux-4.1.6/kernel/cpu.c ---- linux-4.1.6.orig/kernel/cpu.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/cpu.c 2015-09-08 23:49:08.089869160 +0200 +diff -Nur linux-4.1.10.orig/kernel/cgroup.c.orig linux-4.1.10/kernel/cgroup.c.orig +--- linux-4.1.10.orig/kernel/cgroup.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/cgroup.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,5602 @@ ++/* ++ * Generic process-grouping system. ++ * ++ * Based originally on the cpuset system, extracted by Paul Menage ++ * Copyright (C) 2006 Google, Inc ++ * ++ * Notifications support ++ * Copyright (C) 2009 Nokia Corporation ++ * Author: Kirill A. Shutemov ++ * ++ * Copyright notices from the original cpuset code: ++ * -------------------------------------------------- ++ * Copyright (C) 2003 BULL SA. ++ * Copyright (C) 2004-2006 Silicon Graphics, Inc. ++ * ++ * Portions derived from Patrick Mochel's sysfs code. ++ * sysfs is Copyright (c) 2001-3 Patrick Mochel ++ * ++ * 2003-10-10 Written by Simon Derr. ++ * 2003-10-22 Updates by Stephen Hemminger. ++ * 2004 May-July Rework by Paul Jackson. ++ * --------------------------------------------------- ++ * ++ * This file is subject to the terms and conditions of the GNU General Public ++ * License. See the file COPYING in the main directory of the Linux ++ * distribution for more details. ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include <linux/cgroup.h> ++#include <linux/cred.h> ++#include <linux/ctype.h> ++#include <linux/errno.h> ++#include <linux/init_task.h> ++#include <linux/kernel.h> ++#include <linux/list.h> ++#include <linux/magic.h> ++#include <linux/mm.h> ++#include <linux/mutex.h> ++#include <linux/mount.h> ++#include <linux/pagemap.h> ++#include <linux/proc_fs.h> ++#include <linux/rcupdate.h> ++#include <linux/sched.h> ++#include <linux/slab.h> ++#include <linux/spinlock.h> ++#include <linux/rwsem.h> ++#include <linux/string.h> ++#include <linux/sort.h> ++#include <linux/kmod.h> ++#include <linux/delayacct.h> ++#include <linux/cgroupstats.h> ++#include <linux/hashtable.h> ++#include <linux/pid_namespace.h> ++#include <linux/idr.h> ++#include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */ ++#include <linux/kthread.h> ++#include <linux/delay.h> ++ ++#include <linux/atomic.h> ++ ++/* ++ * pidlists linger the following amount before being destroyed. The goal ++ * is avoiding frequent destruction in the middle of consecutive read calls ++ * Expiring in the middle is a performance problem not a correctness one. ++ * 1 sec should be enough. ++ */ ++#define CGROUP_PIDLIST_DESTROY_DELAY HZ ++ ++#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \ ++ MAX_CFTYPE_NAME + 2) ++ ++/* ++ * cgroup_mutex is the master lock. Any modification to cgroup or its ++ * hierarchy must be performed while holding it. ++ * ++ * css_set_rwsem protects task->cgroups pointer, the list of css_set ++ * objects, and the chain of tasks off each css_set. ++ * ++ * These locks are exported if CONFIG_PROVE_RCU so that accessors in ++ * cgroup.h can use them for lockdep annotations. ++ */ ++#ifdef CONFIG_PROVE_RCU ++DEFINE_MUTEX(cgroup_mutex); ++DECLARE_RWSEM(css_set_rwsem); ++EXPORT_SYMBOL_GPL(cgroup_mutex); ++EXPORT_SYMBOL_GPL(css_set_rwsem); ++#else ++static DEFINE_MUTEX(cgroup_mutex); ++static DECLARE_RWSEM(css_set_rwsem); ++#endif ++ ++/* ++ * Protects cgroup_idr and css_idr so that IDs can be released without ++ * grabbing cgroup_mutex. ++ */ ++static DEFINE_SPINLOCK(cgroup_idr_lock); ++ ++/* ++ * Protects cgroup_subsys->release_agent_path. Modifying it also requires ++ * cgroup_mutex. Reading requires either cgroup_mutex or this spinlock. ++ */ ++static DEFINE_SPINLOCK(release_agent_path_lock); ++ ++#define cgroup_assert_mutex_or_rcu_locked() \ ++ rcu_lockdep_assert(rcu_read_lock_held() || \ ++ lockdep_is_held(&cgroup_mutex), \ ++ "cgroup_mutex or RCU read lock required"); ++ ++/* ++ * cgroup destruction makes heavy use of work items and there can be a lot ++ * of concurrent destructions. Use a separate workqueue so that cgroup ++ * destruction work items don't end up filling up max_active of system_wq ++ * which may lead to deadlock. ++ */ ++static struct workqueue_struct *cgroup_destroy_wq; ++ ++/* ++ * pidlist destructions need to be flushed on cgroup destruction. Use a ++ * separate workqueue as flush domain. ++ */ ++static struct workqueue_struct *cgroup_pidlist_destroy_wq; ++ ++/* generate an array of cgroup subsystem pointers */ ++#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys, ++static struct cgroup_subsys *cgroup_subsys[] = { ++#include <linux/cgroup_subsys.h> ++}; ++#undef SUBSYS ++ ++/* array of cgroup subsystem names */ ++#define SUBSYS(_x) [_x ## _cgrp_id] = #_x, ++static const char *cgroup_subsys_name[] = { ++#include <linux/cgroup_subsys.h> ++}; ++#undef SUBSYS ++ ++/* ++ * The default hierarchy, reserved for the subsystems that are otherwise ++ * unattached - it never has more than a single cgroup, and all tasks are ++ * part of that cgroup. ++ */ ++struct cgroup_root cgrp_dfl_root; ++ ++/* ++ * The default hierarchy always exists but is hidden until mounted for the ++ * first time. This is for backward compatibility. ++ */ ++static bool cgrp_dfl_root_visible; ++ ++/* ++ * Set by the boot param of the same name and makes subsystems with NULL ++ * ->dfl_files to use ->legacy_files on the default hierarchy. ++ */ ++static bool cgroup_legacy_files_on_dfl; ++ ++/* some controllers are not supported in the default hierarchy */ ++static unsigned int cgrp_dfl_root_inhibit_ss_mask; ++ ++/* The list of hierarchy roots */ ++ ++static LIST_HEAD(cgroup_roots); ++static int cgroup_root_count; ++ ++/* hierarchy ID allocation and mapping, protected by cgroup_mutex */ ++static DEFINE_IDR(cgroup_hierarchy_idr); ++ ++/* ++ * Assign a monotonically increasing serial number to csses. It guarantees ++ * cgroups with bigger numbers are newer than those with smaller numbers. ++ * Also, as csses are always appended to the parent's ->children list, it ++ * guarantees that sibling csses are always sorted in the ascending serial ++ * number order on the list. Protected by cgroup_mutex. ++ */ ++static u64 css_serial_nr_next = 1; ++ ++/* This flag indicates whether tasks in the fork and exit paths should ++ * check for fork/exit handlers to call. This avoids us having to do ++ * extra work in the fork/exit path if none of the subsystems need to ++ * be called. ++ */ ++static int need_forkexit_callback __read_mostly; ++ ++static struct cftype cgroup_dfl_base_files[]; ++static struct cftype cgroup_legacy_base_files[]; ++ ++static int rebind_subsystems(struct cgroup_root *dst_root, ++ unsigned int ss_mask); ++static int cgroup_destroy_locked(struct cgroup *cgrp); ++static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss, ++ bool visible); ++static void css_release(struct percpu_ref *ref); ++static void kill_css(struct cgroup_subsys_state *css); ++static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[], ++ bool is_add); ++ ++/* IDR wrappers which synchronize using cgroup_idr_lock */ ++static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end, ++ gfp_t gfp_mask) ++{ ++ int ret; ++ ++ idr_preload(gfp_mask); ++ spin_lock_bh(&cgroup_idr_lock); ++ ret = idr_alloc(idr, ptr, start, end, gfp_mask); ++ spin_unlock_bh(&cgroup_idr_lock); ++ idr_preload_end(); ++ return ret; ++} ++ ++static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id) ++{ ++ void *ret; ++ ++ spin_lock_bh(&cgroup_idr_lock); ++ ret = idr_replace(idr, ptr, id); ++ spin_unlock_bh(&cgroup_idr_lock); ++ return ret; ++} ++ ++static void cgroup_idr_remove(struct idr *idr, int id) ++{ ++ spin_lock_bh(&cgroup_idr_lock); ++ idr_remove(idr, id); ++ spin_unlock_bh(&cgroup_idr_lock); ++} ++ ++static struct cgroup *cgroup_parent(struct cgroup *cgrp) ++{ ++ struct cgroup_subsys_state *parent_css = cgrp->self.parent; ++ ++ if (parent_css) ++ return container_of(parent_css, struct cgroup, self); ++ return NULL; ++} ++ ++/** ++ * cgroup_css - obtain a cgroup's css for the specified subsystem ++ * @cgrp: the cgroup of interest ++ * @ss: the subsystem of interest (%NULL returns @cgrp->self) ++ * ++ * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This ++ * function must be called either under cgroup_mutex or rcu_read_lock() and ++ * the caller is responsible for pinning the returned css if it wants to ++ * keep accessing it outside the said locks. This function may return ++ * %NULL if @cgrp doesn't have @subsys_id enabled. ++ */ ++static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, ++ struct cgroup_subsys *ss) ++{ ++ if (ss) ++ return rcu_dereference_check(cgrp->subsys[ss->id], ++ lockdep_is_held(&cgroup_mutex)); ++ else ++ return &cgrp->self; ++} ++ ++/** ++ * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem ++ * @cgrp: the cgroup of interest ++ * @ss: the subsystem of interest (%NULL returns @cgrp->self) ++ * ++ * Similar to cgroup_css() but returns the effctive css, which is defined ++ * as the matching css of the nearest ancestor including self which has @ss ++ * enabled. If @ss is associated with the hierarchy @cgrp is on, this ++ * function is guaranteed to return non-NULL css. ++ */ ++static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, ++ struct cgroup_subsys *ss) ++{ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ if (!ss) ++ return &cgrp->self; ++ ++ if (!(cgrp->root->subsys_mask & (1 << ss->id))) ++ return NULL; ++ ++ /* ++ * This function is used while updating css associations and thus ++ * can't test the csses directly. Use ->child_subsys_mask. ++ */ ++ while (cgroup_parent(cgrp) && ++ !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ss->id))) ++ cgrp = cgroup_parent(cgrp); ++ ++ return cgroup_css(cgrp, ss); ++} ++ ++/** ++ * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem ++ * @cgrp: the cgroup of interest ++ * @ss: the subsystem of interest ++ * ++ * Find and get the effective css of @cgrp for @ss. The effective css is ++ * defined as the matching css of the nearest ancestor including self which ++ * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on, ++ * the root css is returned, so this function always returns a valid css. ++ * The returned css must be put using css_put(). ++ */ ++struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, ++ struct cgroup_subsys *ss) ++{ ++ struct cgroup_subsys_state *css; ++ ++ rcu_read_lock(); ++ ++ do { ++ css = cgroup_css(cgrp, ss); ++ ++ if (css && css_tryget_online(css)) ++ goto out_unlock; ++ cgrp = cgroup_parent(cgrp); ++ } while (cgrp); ++ ++ css = init_css_set.subsys[ss->id]; ++ css_get(css); ++out_unlock: ++ rcu_read_unlock(); ++ return css; ++} ++ ++/* convenient tests for these bits */ ++static inline bool cgroup_is_dead(const struct cgroup *cgrp) ++{ ++ return !(cgrp->self.flags & CSS_ONLINE); ++} ++ ++struct cgroup_subsys_state *of_css(struct kernfs_open_file *of) ++{ ++ struct cgroup *cgrp = of->kn->parent->priv; ++ struct cftype *cft = of_cft(of); ++ ++ /* ++ * This is open and unprotected implementation of cgroup_css(). ++ * seq_css() is only called from a kernfs file operation which has ++ * an active reference on the file. Because all the subsystem ++ * files are drained before a css is disassociated with a cgroup, ++ * the matching css from the cgroup's subsys table is guaranteed to ++ * be and stay valid until the enclosing operation is complete. ++ */ ++ if (cft->ss) ++ return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); ++ else ++ return &cgrp->self; ++} ++EXPORT_SYMBOL_GPL(of_css); ++ ++/** ++ * cgroup_is_descendant - test ancestry ++ * @cgrp: the cgroup to be tested ++ * @ancestor: possible ancestor of @cgrp ++ * ++ * Test whether @cgrp is a descendant of @ancestor. It also returns %true ++ * if @cgrp == @ancestor. This function is safe to call as long as @cgrp ++ * and @ancestor are accessible. ++ */ ++bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor) ++{ ++ while (cgrp) { ++ if (cgrp == ancestor) ++ return true; ++ cgrp = cgroup_parent(cgrp); ++ } ++ return false; ++} ++ ++static int notify_on_release(const struct cgroup *cgrp) ++{ ++ return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); ++} ++ ++/** ++ * for_each_css - iterate all css's of a cgroup ++ * @css: the iteration cursor ++ * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end ++ * @cgrp: the target cgroup to iterate css's of ++ * ++ * Should be called under cgroup_[tree_]mutex. ++ */ ++#define for_each_css(css, ssid, cgrp) \ ++ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ ++ if (!((css) = rcu_dereference_check( \ ++ (cgrp)->subsys[(ssid)], \ ++ lockdep_is_held(&cgroup_mutex)))) { } \ ++ else ++ ++/** ++ * for_each_e_css - iterate all effective css's of a cgroup ++ * @css: the iteration cursor ++ * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end ++ * @cgrp: the target cgroup to iterate css's of ++ * ++ * Should be called under cgroup_[tree_]mutex. ++ */ ++#define for_each_e_css(css, ssid, cgrp) \ ++ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ ++ if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \ ++ ; \ ++ else ++ ++/** ++ * for_each_subsys - iterate all enabled cgroup subsystems ++ * @ss: the iteration cursor ++ * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end ++ */ ++#define for_each_subsys(ss, ssid) \ ++ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT && \ ++ (((ss) = cgroup_subsys[ssid]) || true); (ssid)++) ++ ++/* iterate across the hierarchies */ ++#define for_each_root(root) \ ++ list_for_each_entry((root), &cgroup_roots, root_list) ++ ++/* iterate over child cgrps, lock should be held throughout iteration */ ++#define cgroup_for_each_live_child(child, cgrp) \ ++ list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \ ++ if (({ lockdep_assert_held(&cgroup_mutex); \ ++ cgroup_is_dead(child); })) \ ++ ; \ ++ else ++ ++static void cgroup_release_agent(struct work_struct *work); ++static void check_for_release(struct cgroup *cgrp); ++ ++/* ++ * A cgroup can be associated with multiple css_sets as different tasks may ++ * belong to different cgroups on different hierarchies. In the other ++ * direction, a css_set is naturally associated with multiple cgroups. ++ * This M:N relationship is represented by the following link structure ++ * which exists for each association and allows traversing the associations ++ * from both sides. ++ */ ++struct cgrp_cset_link { ++ /* the cgroup and css_set this link associates */ ++ struct cgroup *cgrp; ++ struct css_set *cset; ++ ++ /* list of cgrp_cset_links anchored at cgrp->cset_links */ ++ struct list_head cset_link; ++ ++ /* list of cgrp_cset_links anchored at css_set->cgrp_links */ ++ struct list_head cgrp_link; ++}; ++ ++/* ++ * The default css_set - used by init and its children prior to any ++ * hierarchies being mounted. It contains a pointer to the root state ++ * for each subsystem. Also used to anchor the list of css_sets. Not ++ * reference-counted, to improve performance when child cgroups ++ * haven't been created. ++ */ ++struct css_set init_css_set = { ++ .refcount = ATOMIC_INIT(1), ++ .cgrp_links = LIST_HEAD_INIT(init_css_set.cgrp_links), ++ .tasks = LIST_HEAD_INIT(init_css_set.tasks), ++ .mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks), ++ .mg_preload_node = LIST_HEAD_INIT(init_css_set.mg_preload_node), ++ .mg_node = LIST_HEAD_INIT(init_css_set.mg_node), ++}; ++ ++static int css_set_count = 1; /* 1 for init_css_set */ ++ ++/** ++ * cgroup_update_populated - updated populated count of a cgroup ++ * @cgrp: the target cgroup ++ * @populated: inc or dec populated count ++ * ++ * @cgrp is either getting the first task (css_set) or losing the last. ++ * Update @cgrp->populated_cnt accordingly. The count is propagated ++ * towards root so that a given cgroup's populated_cnt is zero iff the ++ * cgroup and all its descendants are empty. ++ * ++ * @cgrp's interface file "cgroup.populated" is zero if ++ * @cgrp->populated_cnt is zero and 1 otherwise. When @cgrp->populated_cnt ++ * changes from or to zero, userland is notified that the content of the ++ * interface file has changed. This can be used to detect when @cgrp and ++ * its descendants become populated or empty. ++ */ ++static void cgroup_update_populated(struct cgroup *cgrp, bool populated) ++{ ++ lockdep_assert_held(&css_set_rwsem); ++ ++ do { ++ bool trigger; ++ ++ if (populated) ++ trigger = !cgrp->populated_cnt++; ++ else ++ trigger = !--cgrp->populated_cnt; ++ ++ if (!trigger) ++ break; ++ ++ if (cgrp->populated_kn) ++ kernfs_notify(cgrp->populated_kn); ++ cgrp = cgroup_parent(cgrp); ++ } while (cgrp); ++} ++ ++/* ++ * hash table for cgroup groups. This improves the performance to find ++ * an existing css_set. This hash doesn't (currently) take into ++ * account cgroups in empty hierarchies. ++ */ ++#define CSS_SET_HASH_BITS 7 ++static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS); ++ ++static unsigned long css_set_hash(struct cgroup_subsys_state *css[]) ++{ ++ unsigned long key = 0UL; ++ struct cgroup_subsys *ss; ++ int i; ++ ++ for_each_subsys(ss, i) ++ key += (unsigned long)css[i]; ++ key = (key >> 16) ^ key; ++ ++ return key; ++} ++ ++static void put_css_set_locked(struct css_set *cset) ++{ ++ struct cgrp_cset_link *link, *tmp_link; ++ struct cgroup_subsys *ss; ++ int ssid; ++ ++ lockdep_assert_held(&css_set_rwsem); ++ ++ if (!atomic_dec_and_test(&cset->refcount)) ++ return; ++ ++ /* This css_set is dead. unlink it and release cgroup refcounts */ ++ for_each_subsys(ss, ssid) ++ list_del(&cset->e_cset_node[ssid]); ++ hash_del(&cset->hlist); ++ css_set_count--; ++ ++ list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) { ++ struct cgroup *cgrp = link->cgrp; ++ ++ list_del(&link->cset_link); ++ list_del(&link->cgrp_link); ++ ++ /* @cgrp can't go away while we're holding css_set_rwsem */ ++ if (list_empty(&cgrp->cset_links)) { ++ cgroup_update_populated(cgrp, false); ++ check_for_release(cgrp); ++ } ++ ++ kfree(link); ++ } ++ ++ kfree_rcu(cset, rcu_head); ++} ++ ++static void put_css_set(struct css_set *cset) ++{ ++ /* ++ * Ensure that the refcount doesn't hit zero while any readers ++ * can see it. Similar to atomic_dec_and_lock(), but for an ++ * rwlock ++ */ ++ if (atomic_add_unless(&cset->refcount, -1, 1)) ++ return; ++ ++ down_write(&css_set_rwsem); ++ put_css_set_locked(cset); ++ up_write(&css_set_rwsem); ++} ++ ++/* ++ * refcounted get/put for css_set objects ++ */ ++static inline void get_css_set(struct css_set *cset) ++{ ++ atomic_inc(&cset->refcount); ++} ++ ++/** ++ * compare_css_sets - helper function for find_existing_css_set(). ++ * @cset: candidate css_set being tested ++ * @old_cset: existing css_set for a task ++ * @new_cgrp: cgroup that's being entered by the task ++ * @template: desired set of css pointers in css_set (pre-calculated) ++ * ++ * Returns true if "cset" matches "old_cset" except for the hierarchy ++ * which "new_cgrp" belongs to, for which it should match "new_cgrp". ++ */ ++static bool compare_css_sets(struct css_set *cset, ++ struct css_set *old_cset, ++ struct cgroup *new_cgrp, ++ struct cgroup_subsys_state *template[]) ++{ ++ struct list_head *l1, *l2; ++ ++ /* ++ * On the default hierarchy, there can be csets which are ++ * associated with the same set of cgroups but different csses. ++ * Let's first ensure that csses match. ++ */ ++ if (memcmp(template, cset->subsys, sizeof(cset->subsys))) ++ return false; ++ ++ /* ++ * Compare cgroup pointers in order to distinguish between ++ * different cgroups in hierarchies. As different cgroups may ++ * share the same effective css, this comparison is always ++ * necessary. ++ */ ++ l1 = &cset->cgrp_links; ++ l2 = &old_cset->cgrp_links; ++ while (1) { ++ struct cgrp_cset_link *link1, *link2; ++ struct cgroup *cgrp1, *cgrp2; ++ ++ l1 = l1->next; ++ l2 = l2->next; ++ /* See if we reached the end - both lists are equal length. */ ++ if (l1 == &cset->cgrp_links) { ++ BUG_ON(l2 != &old_cset->cgrp_links); ++ break; ++ } else { ++ BUG_ON(l2 == &old_cset->cgrp_links); ++ } ++ /* Locate the cgroups associated with these links. */ ++ link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link); ++ link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link); ++ cgrp1 = link1->cgrp; ++ cgrp2 = link2->cgrp; ++ /* Hierarchies should be linked in the same order. */ ++ BUG_ON(cgrp1->root != cgrp2->root); ++ ++ /* ++ * If this hierarchy is the hierarchy of the cgroup ++ * that's changing, then we need to check that this ++ * css_set points to the new cgroup; if it's any other ++ * hierarchy, then this css_set should point to the ++ * same cgroup as the old css_set. ++ */ ++ if (cgrp1->root == new_cgrp->root) { ++ if (cgrp1 != new_cgrp) ++ return false; ++ } else { ++ if (cgrp1 != cgrp2) ++ return false; ++ } ++ } ++ return true; ++} ++ ++/** ++ * find_existing_css_set - init css array and find the matching css_set ++ * @old_cset: the css_set that we're using before the cgroup transition ++ * @cgrp: the cgroup that we're moving into ++ * @template: out param for the new set of csses, should be clear on entry ++ */ ++static struct css_set *find_existing_css_set(struct css_set *old_cset, ++ struct cgroup *cgrp, ++ struct cgroup_subsys_state *template[]) ++{ ++ struct cgroup_root *root = cgrp->root; ++ struct cgroup_subsys *ss; ++ struct css_set *cset; ++ unsigned long key; ++ int i; ++ ++ /* ++ * Build the set of subsystem state objects that we want to see in the ++ * new css_set. while subsystems can change globally, the entries here ++ * won't change, so no need for locking. ++ */ ++ for_each_subsys(ss, i) { ++ if (root->subsys_mask & (1UL << i)) { ++ /* ++ * @ss is in this hierarchy, so we want the ++ * effective css from @cgrp. ++ */ ++ template[i] = cgroup_e_css(cgrp, ss); ++ } else { ++ /* ++ * @ss is not in this hierarchy, so we don't want ++ * to change the css. ++ */ ++ template[i] = old_cset->subsys[i]; ++ } ++ } ++ ++ key = css_set_hash(template); ++ hash_for_each_possible(css_set_table, cset, hlist, key) { ++ if (!compare_css_sets(cset, old_cset, cgrp, template)) ++ continue; ++ ++ /* This css_set matches what we need */ ++ return cset; ++ } ++ ++ /* No existing cgroup group matched */ ++ return NULL; ++} ++ ++static void free_cgrp_cset_links(struct list_head *links_to_free) ++{ ++ struct cgrp_cset_link *link, *tmp_link; ++ ++ list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) { ++ list_del(&link->cset_link); ++ kfree(link); ++ } ++} ++ ++/** ++ * allocate_cgrp_cset_links - allocate cgrp_cset_links ++ * @count: the number of links to allocate ++ * @tmp_links: list_head the allocated links are put on ++ * ++ * Allocate @count cgrp_cset_link structures and chain them on @tmp_links ++ * through ->cset_link. Returns 0 on success or -errno. ++ */ ++static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links) ++{ ++ struct cgrp_cset_link *link; ++ int i; ++ ++ INIT_LIST_HEAD(tmp_links); ++ ++ for (i = 0; i < count; i++) { ++ link = kzalloc(sizeof(*link), GFP_KERNEL); ++ if (!link) { ++ free_cgrp_cset_links(tmp_links); ++ return -ENOMEM; ++ } ++ list_add(&link->cset_link, tmp_links); ++ } ++ return 0; ++} ++ ++/** ++ * link_css_set - a helper function to link a css_set to a cgroup ++ * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links() ++ * @cset: the css_set to be linked ++ * @cgrp: the destination cgroup ++ */ ++static void link_css_set(struct list_head *tmp_links, struct css_set *cset, ++ struct cgroup *cgrp) ++{ ++ struct cgrp_cset_link *link; ++ ++ BUG_ON(list_empty(tmp_links)); ++ ++ if (cgroup_on_dfl(cgrp)) ++ cset->dfl_cgrp = cgrp; ++ ++ link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link); ++ link->cset = cset; ++ link->cgrp = cgrp; ++ ++ if (list_empty(&cgrp->cset_links)) ++ cgroup_update_populated(cgrp, true); ++ list_move(&link->cset_link, &cgrp->cset_links); ++ ++ /* ++ * Always add links to the tail of the list so that the list ++ * is sorted by order of hierarchy creation ++ */ ++ list_add_tail(&link->cgrp_link, &cset->cgrp_links); ++} ++ ++/** ++ * find_css_set - return a new css_set with one cgroup updated ++ * @old_cset: the baseline css_set ++ * @cgrp: the cgroup to be updated ++ * ++ * Return a new css_set that's equivalent to @old_cset, but with @cgrp ++ * substituted into the appropriate hierarchy. ++ */ ++static struct css_set *find_css_set(struct css_set *old_cset, ++ struct cgroup *cgrp) ++{ ++ struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { }; ++ struct css_set *cset; ++ struct list_head tmp_links; ++ struct cgrp_cset_link *link; ++ struct cgroup_subsys *ss; ++ unsigned long key; ++ int ssid; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* First see if we already have a cgroup group that matches ++ * the desired set */ ++ down_read(&css_set_rwsem); ++ cset = find_existing_css_set(old_cset, cgrp, template); ++ if (cset) ++ get_css_set(cset); ++ up_read(&css_set_rwsem); ++ ++ if (cset) ++ return cset; ++ ++ cset = kzalloc(sizeof(*cset), GFP_KERNEL); ++ if (!cset) ++ return NULL; ++ ++ /* Allocate all the cgrp_cset_link objects that we'll need */ ++ if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) { ++ kfree(cset); ++ return NULL; ++ } ++ ++ atomic_set(&cset->refcount, 1); ++ INIT_LIST_HEAD(&cset->cgrp_links); ++ INIT_LIST_HEAD(&cset->tasks); ++ INIT_LIST_HEAD(&cset->mg_tasks); ++ INIT_LIST_HEAD(&cset->mg_preload_node); ++ INIT_LIST_HEAD(&cset->mg_node); ++ INIT_HLIST_NODE(&cset->hlist); ++ ++ /* Copy the set of subsystem state objects generated in ++ * find_existing_css_set() */ ++ memcpy(cset->subsys, template, sizeof(cset->subsys)); ++ ++ down_write(&css_set_rwsem); ++ /* Add reference counts and links from the new css_set. */ ++ list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) { ++ struct cgroup *c = link->cgrp; ++ ++ if (c->root == cgrp->root) ++ c = cgrp; ++ link_css_set(&tmp_links, cset, c); ++ } ++ ++ BUG_ON(!list_empty(&tmp_links)); ++ ++ css_set_count++; ++ ++ /* Add @cset to the hash table */ ++ key = css_set_hash(cset->subsys); ++ hash_add(css_set_table, &cset->hlist, key); ++ ++ for_each_subsys(ss, ssid) ++ list_add_tail(&cset->e_cset_node[ssid], ++ &cset->subsys[ssid]->cgroup->e_csets[ssid]); ++ ++ up_write(&css_set_rwsem); ++ ++ return cset; ++} ++ ++static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root) ++{ ++ struct cgroup *root_cgrp = kf_root->kn->priv; ++ ++ return root_cgrp->root; ++} ++ ++static int cgroup_init_root_id(struct cgroup_root *root) ++{ ++ int id; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL); ++ if (id < 0) ++ return id; ++ ++ root->hierarchy_id = id; ++ return 0; ++} ++ ++static void cgroup_exit_root_id(struct cgroup_root *root) ++{ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ if (root->hierarchy_id) { ++ idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id); ++ root->hierarchy_id = 0; ++ } ++} ++ ++static void cgroup_free_root(struct cgroup_root *root) ++{ ++ if (root) { ++ /* hierarhcy ID shoulid already have been released */ ++ WARN_ON_ONCE(root->hierarchy_id); ++ ++ idr_destroy(&root->cgroup_idr); ++ kfree(root); ++ } ++} ++ ++static void cgroup_destroy_root(struct cgroup_root *root) ++{ ++ struct cgroup *cgrp = &root->cgrp; ++ struct cgrp_cset_link *link, *tmp_link; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ BUG_ON(atomic_read(&root->nr_cgrps)); ++ BUG_ON(!list_empty(&cgrp->self.children)); ++ ++ /* Rebind all subsystems back to the default hierarchy */ ++ rebind_subsystems(&cgrp_dfl_root, root->subsys_mask); ++ ++ /* ++ * Release all the links from cset_links to this hierarchy's ++ * root cgroup ++ */ ++ down_write(&css_set_rwsem); ++ ++ list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { ++ list_del(&link->cset_link); ++ list_del(&link->cgrp_link); ++ kfree(link); ++ } ++ up_write(&css_set_rwsem); ++ ++ if (!list_empty(&root->root_list)) { ++ list_del(&root->root_list); ++ cgroup_root_count--; ++ } ++ ++ cgroup_exit_root_id(root); ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ kernfs_destroy_root(root->kf_root); ++ cgroup_free_root(root); ++} ++ ++/* look up cgroup associated with given css_set on the specified hierarchy */ ++static struct cgroup *cset_cgroup_from_root(struct css_set *cset, ++ struct cgroup_root *root) ++{ ++ struct cgroup *res = NULL; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ lockdep_assert_held(&css_set_rwsem); ++ ++ if (cset == &init_css_set) { ++ res = &root->cgrp; ++ } else { ++ struct cgrp_cset_link *link; ++ ++ list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { ++ struct cgroup *c = link->cgrp; ++ ++ if (c->root == root) { ++ res = c; ++ break; ++ } ++ } ++ } ++ ++ BUG_ON(!res); ++ return res; ++} ++ ++/* ++ * Return the cgroup for "task" from the given hierarchy. Must be ++ * called with cgroup_mutex and css_set_rwsem held. ++ */ ++static struct cgroup *task_cgroup_from_root(struct task_struct *task, ++ struct cgroup_root *root) ++{ ++ /* ++ * No need to lock the task - since we hold cgroup_mutex the ++ * task can't change groups, so the only thing that can happen ++ * is that it exits and its css is set back to init_css_set. ++ */ ++ return cset_cgroup_from_root(task_css_set(task), root); ++} ++ ++/* ++ * A task must hold cgroup_mutex to modify cgroups. ++ * ++ * Any task can increment and decrement the count field without lock. ++ * So in general, code holding cgroup_mutex can't rely on the count ++ * field not changing. However, if the count goes to zero, then only ++ * cgroup_attach_task() can increment it again. Because a count of zero ++ * means that no tasks are currently attached, therefore there is no ++ * way a task attached to that cgroup can fork (the other way to ++ * increment the count). So code holding cgroup_mutex can safely ++ * assume that if the count is zero, it will stay zero. Similarly, if ++ * a task holds cgroup_mutex on a cgroup with zero count, it ++ * knows that the cgroup won't be removed, as cgroup_rmdir() ++ * needs that mutex. ++ * ++ * A cgroup can only be deleted if both its 'count' of using tasks ++ * is zero, and its list of 'children' cgroups is empty. Since all ++ * tasks in the system use _some_ cgroup, and since there is always at ++ * least one task in the system (init, pid == 1), therefore, root cgroup ++ * always has either children cgroups and/or using tasks. So we don't ++ * need a special hack to ensure that root cgroup cannot be deleted. ++ * ++ * P.S. One more locking exception. RCU is used to guard the ++ * update of a tasks cgroup pointer by cgroup_attach_task() ++ */ ++ ++static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask); ++static struct kernfs_syscall_ops cgroup_kf_syscall_ops; ++static const struct file_operations proc_cgroupstats_operations; ++ ++static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft, ++ char *buf) ++{ ++ if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) && ++ !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) ++ snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s", ++ cft->ss->name, cft->name); ++ else ++ strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX); ++ return buf; ++} ++ ++/** ++ * cgroup_file_mode - deduce file mode of a control file ++ * @cft: the control file in question ++ * ++ * returns cft->mode if ->mode is not 0 ++ * returns S_IRUGO|S_IWUSR if it has both a read and a write handler ++ * returns S_IRUGO if it has only a read handler ++ * returns S_IWUSR if it has only a write hander ++ */ ++static umode_t cgroup_file_mode(const struct cftype *cft) ++{ ++ umode_t mode = 0; ++ ++ if (cft->mode) ++ return cft->mode; ++ ++ if (cft->read_u64 || cft->read_s64 || cft->seq_show) ++ mode |= S_IRUGO; ++ ++ if (cft->write_u64 || cft->write_s64 || cft->write) ++ mode |= S_IWUSR; ++ ++ return mode; ++} ++ ++static void cgroup_get(struct cgroup *cgrp) ++{ ++ WARN_ON_ONCE(cgroup_is_dead(cgrp)); ++ css_get(&cgrp->self); ++} ++ ++static bool cgroup_tryget(struct cgroup *cgrp) ++{ ++ return css_tryget(&cgrp->self); ++} ++ ++static void cgroup_put(struct cgroup *cgrp) ++{ ++ css_put(&cgrp->self); ++} ++ ++/** ++ * cgroup_calc_child_subsys_mask - calculate child_subsys_mask ++ * @cgrp: the target cgroup ++ * @subtree_control: the new subtree_control mask to consider ++ * ++ * On the default hierarchy, a subsystem may request other subsystems to be ++ * enabled together through its ->depends_on mask. In such cases, more ++ * subsystems than specified in "cgroup.subtree_control" may be enabled. ++ * ++ * This function calculates which subsystems need to be enabled if ++ * @subtree_control is to be applied to @cgrp. The returned mask is always ++ * a superset of @subtree_control and follows the usual hierarchy rules. ++ */ ++static unsigned int cgroup_calc_child_subsys_mask(struct cgroup *cgrp, ++ unsigned int subtree_control) ++{ ++ struct cgroup *parent = cgroup_parent(cgrp); ++ unsigned int cur_ss_mask = subtree_control; ++ struct cgroup_subsys *ss; ++ int ssid; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ if (!cgroup_on_dfl(cgrp)) ++ return cur_ss_mask; ++ ++ while (true) { ++ unsigned int new_ss_mask = cur_ss_mask; ++ ++ for_each_subsys(ss, ssid) ++ if (cur_ss_mask & (1 << ssid)) ++ new_ss_mask |= ss->depends_on; ++ ++ /* ++ * Mask out subsystems which aren't available. This can ++ * happen only if some depended-upon subsystems were bound ++ * to non-default hierarchies. ++ */ ++ if (parent) ++ new_ss_mask &= parent->child_subsys_mask; ++ else ++ new_ss_mask &= cgrp->root->subsys_mask; ++ ++ if (new_ss_mask == cur_ss_mask) ++ break; ++ cur_ss_mask = new_ss_mask; ++ } ++ ++ return cur_ss_mask; ++} ++ ++/** ++ * cgroup_refresh_child_subsys_mask - update child_subsys_mask ++ * @cgrp: the target cgroup ++ * ++ * Update @cgrp->child_subsys_mask according to the current ++ * @cgrp->subtree_control using cgroup_calc_child_subsys_mask(). ++ */ ++static void cgroup_refresh_child_subsys_mask(struct cgroup *cgrp) ++{ ++ cgrp->child_subsys_mask = ++ cgroup_calc_child_subsys_mask(cgrp, cgrp->subtree_control); ++} ++ ++/** ++ * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods ++ * @kn: the kernfs_node being serviced ++ * ++ * This helper undoes cgroup_kn_lock_live() and should be invoked before ++ * the method finishes if locking succeeded. Note that once this function ++ * returns the cgroup returned by cgroup_kn_lock_live() may become ++ * inaccessible any time. If the caller intends to continue to access the ++ * cgroup, it should pin it before invoking this function. ++ */ ++static void cgroup_kn_unlock(struct kernfs_node *kn) ++{ ++ struct cgroup *cgrp; ++ ++ if (kernfs_type(kn) == KERNFS_DIR) ++ cgrp = kn->priv; ++ else ++ cgrp = kn->parent->priv; ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ kernfs_unbreak_active_protection(kn); ++ cgroup_put(cgrp); ++} ++ ++/** ++ * cgroup_kn_lock_live - locking helper for cgroup kernfs methods ++ * @kn: the kernfs_node being serviced ++ * ++ * This helper is to be used by a cgroup kernfs method currently servicing ++ * @kn. It breaks the active protection, performs cgroup locking and ++ * verifies that the associated cgroup is alive. Returns the cgroup if ++ * alive; otherwise, %NULL. A successful return should be undone by a ++ * matching cgroup_kn_unlock() invocation. ++ * ++ * Any cgroup kernfs method implementation which requires locking the ++ * associated cgroup should use this helper. It avoids nesting cgroup ++ * locking under kernfs active protection and allows all kernfs operations ++ * including self-removal. ++ */ ++static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn) ++{ ++ struct cgroup *cgrp; ++ ++ if (kernfs_type(kn) == KERNFS_DIR) ++ cgrp = kn->priv; ++ else ++ cgrp = kn->parent->priv; ++ ++ /* ++ * We're gonna grab cgroup_mutex which nests outside kernfs ++ * active_ref. cgroup liveliness check alone provides enough ++ * protection against removal. Ensure @cgrp stays accessible and ++ * break the active_ref protection. ++ */ ++ if (!cgroup_tryget(cgrp)) ++ return NULL; ++ kernfs_break_active_protection(kn); ++ ++ mutex_lock(&cgroup_mutex); ++ ++ if (!cgroup_is_dead(cgrp)) ++ return cgrp; ++ ++ cgroup_kn_unlock(kn); ++ return NULL; ++} ++ ++static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) ++{ ++ char name[CGROUP_FILE_NAME_MAX]; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); ++} ++ ++/** ++ * cgroup_clear_dir - remove subsys files in a cgroup directory ++ * @cgrp: target cgroup ++ * @subsys_mask: mask of the subsystem ids whose files should be removed ++ */ ++static void cgroup_clear_dir(struct cgroup *cgrp, unsigned int subsys_mask) ++{ ++ struct cgroup_subsys *ss; ++ int i; ++ ++ for_each_subsys(ss, i) { ++ struct cftype *cfts; ++ ++ if (!(subsys_mask & (1 << i))) ++ continue; ++ list_for_each_entry(cfts, &ss->cfts, node) ++ cgroup_addrm_files(cgrp, cfts, false); ++ } ++} ++ ++static int rebind_subsystems(struct cgroup_root *dst_root, unsigned int ss_mask) ++{ ++ struct cgroup_subsys *ss; ++ unsigned int tmp_ss_mask; ++ int ssid, i, ret; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ for_each_subsys(ss, ssid) { ++ if (!(ss_mask & (1 << ssid))) ++ continue; ++ ++ /* if @ss has non-root csses attached to it, can't move */ ++ if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss))) ++ return -EBUSY; ++ ++ /* can't move between two non-dummy roots either */ ++ if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root) ++ return -EBUSY; ++ } ++ ++ /* skip creating root files on dfl_root for inhibited subsystems */ ++ tmp_ss_mask = ss_mask; ++ if (dst_root == &cgrp_dfl_root) ++ tmp_ss_mask &= ~cgrp_dfl_root_inhibit_ss_mask; ++ ++ ret = cgroup_populate_dir(&dst_root->cgrp, tmp_ss_mask); ++ if (ret) { ++ if (dst_root != &cgrp_dfl_root) ++ return ret; ++ ++ /* ++ * Rebinding back to the default root is not allowed to ++ * fail. Using both default and non-default roots should ++ * be rare. Moving subsystems back and forth even more so. ++ * Just warn about it and continue. ++ */ ++ if (cgrp_dfl_root_visible) { ++ pr_warn("failed to create files (%d) while rebinding 0x%x to default root\n", ++ ret, ss_mask); ++ pr_warn("you may retry by moving them to a different hierarchy and unbinding\n"); ++ } ++ } ++ ++ /* ++ * Nothing can fail from this point on. Remove files for the ++ * removed subsystems and rebind each subsystem. ++ */ ++ for_each_subsys(ss, ssid) ++ if (ss_mask & (1 << ssid)) ++ cgroup_clear_dir(&ss->root->cgrp, 1 << ssid); ++ ++ for_each_subsys(ss, ssid) { ++ struct cgroup_root *src_root; ++ struct cgroup_subsys_state *css; ++ struct css_set *cset; ++ ++ if (!(ss_mask & (1 << ssid))) ++ continue; ++ ++ src_root = ss->root; ++ css = cgroup_css(&src_root->cgrp, ss); ++ ++ WARN_ON(!css || cgroup_css(&dst_root->cgrp, ss)); ++ ++ RCU_INIT_POINTER(src_root->cgrp.subsys[ssid], NULL); ++ rcu_assign_pointer(dst_root->cgrp.subsys[ssid], css); ++ ss->root = dst_root; ++ css->cgroup = &dst_root->cgrp; ++ ++ down_write(&css_set_rwsem); ++ hash_for_each(css_set_table, i, cset, hlist) ++ list_move_tail(&cset->e_cset_node[ss->id], ++ &dst_root->cgrp.e_csets[ss->id]); ++ up_write(&css_set_rwsem); ++ ++ src_root->subsys_mask &= ~(1 << ssid); ++ src_root->cgrp.subtree_control &= ~(1 << ssid); ++ cgroup_refresh_child_subsys_mask(&src_root->cgrp); ++ ++ /* default hierarchy doesn't enable controllers by default */ ++ dst_root->subsys_mask |= 1 << ssid; ++ if (dst_root != &cgrp_dfl_root) { ++ dst_root->cgrp.subtree_control |= 1 << ssid; ++ cgroup_refresh_child_subsys_mask(&dst_root->cgrp); ++ } ++ ++ if (ss->bind) ++ ss->bind(css); ++ } ++ ++ kernfs_activate(dst_root->cgrp.kn); ++ return 0; ++} ++ ++static int cgroup_show_options(struct seq_file *seq, ++ struct kernfs_root *kf_root) ++{ ++ struct cgroup_root *root = cgroup_root_from_kf(kf_root); ++ struct cgroup_subsys *ss; ++ int ssid; ++ ++ for_each_subsys(ss, ssid) ++ if (root->subsys_mask & (1 << ssid)) ++ seq_show_option(seq, ss->name, NULL); ++ if (root->flags & CGRP_ROOT_NOPREFIX) ++ seq_puts(seq, ",noprefix"); ++ if (root->flags & CGRP_ROOT_XATTR) ++ seq_puts(seq, ",xattr"); ++ ++ spin_lock(&release_agent_path_lock); ++ if (strlen(root->release_agent_path)) ++ seq_show_option(seq, "release_agent", ++ root->release_agent_path); ++ spin_unlock(&release_agent_path_lock); ++ ++ if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags)) ++ seq_puts(seq, ",clone_children"); ++ if (strlen(root->name)) ++ seq_show_option(seq, "name", root->name); ++ return 0; ++} ++ ++struct cgroup_sb_opts { ++ unsigned int subsys_mask; ++ unsigned int flags; ++ char *release_agent; ++ bool cpuset_clone_children; ++ char *name; ++ /* User explicitly requested empty subsystem */ ++ bool none; ++}; ++ ++static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts) ++{ ++ char *token, *o = data; ++ bool all_ss = false, one_ss = false; ++ unsigned int mask = -1U; ++ struct cgroup_subsys *ss; ++ int nr_opts = 0; ++ int i; ++ ++#ifdef CONFIG_CPUSETS ++ mask = ~(1U << cpuset_cgrp_id); ++#endif ++ ++ memset(opts, 0, sizeof(*opts)); ++ ++ while ((token = strsep(&o, ",")) != NULL) { ++ nr_opts++; ++ ++ if (!*token) ++ return -EINVAL; ++ if (!strcmp(token, "none")) { ++ /* Explicitly have no subsystems */ ++ opts->none = true; ++ continue; ++ } ++ if (!strcmp(token, "all")) { ++ /* Mutually exclusive option 'all' + subsystem name */ ++ if (one_ss) ++ return -EINVAL; ++ all_ss = true; ++ continue; ++ } ++ if (!strcmp(token, "__DEVEL__sane_behavior")) { ++ opts->flags |= CGRP_ROOT_SANE_BEHAVIOR; ++ continue; ++ } ++ if (!strcmp(token, "noprefix")) { ++ opts->flags |= CGRP_ROOT_NOPREFIX; ++ continue; ++ } ++ if (!strcmp(token, "clone_children")) { ++ opts->cpuset_clone_children = true; ++ continue; ++ } ++ if (!strcmp(token, "xattr")) { ++ opts->flags |= CGRP_ROOT_XATTR; ++ continue; ++ } ++ if (!strncmp(token, "release_agent=", 14)) { ++ /* Specifying two release agents is forbidden */ ++ if (opts->release_agent) ++ return -EINVAL; ++ opts->release_agent = ++ kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL); ++ if (!opts->release_agent) ++ return -ENOMEM; ++ continue; ++ } ++ if (!strncmp(token, "name=", 5)) { ++ const char *name = token + 5; ++ /* Can't specify an empty name */ ++ if (!strlen(name)) ++ return -EINVAL; ++ /* Must match [\w.-]+ */ ++ for (i = 0; i < strlen(name); i++) { ++ char c = name[i]; ++ if (isalnum(c)) ++ continue; ++ if ((c == '.') || (c == '-') || (c == '_')) ++ continue; ++ return -EINVAL; ++ } ++ /* Specifying two names is forbidden */ ++ if (opts->name) ++ return -EINVAL; ++ opts->name = kstrndup(name, ++ MAX_CGROUP_ROOT_NAMELEN - 1, ++ GFP_KERNEL); ++ if (!opts->name) ++ return -ENOMEM; ++ ++ continue; ++ } ++ ++ for_each_subsys(ss, i) { ++ if (strcmp(token, ss->name)) ++ continue; ++ if (ss->disabled) ++ continue; ++ ++ /* Mutually exclusive option 'all' + subsystem name */ ++ if (all_ss) ++ return -EINVAL; ++ opts->subsys_mask |= (1 << i); ++ one_ss = true; ++ ++ break; ++ } ++ if (i == CGROUP_SUBSYS_COUNT) ++ return -ENOENT; ++ } ++ ++ if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) { ++ pr_warn("sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n"); ++ if (nr_opts != 1) { ++ pr_err("sane_behavior: no other mount options allowed\n"); ++ return -EINVAL; ++ } ++ return 0; ++ } ++ ++ /* ++ * If the 'all' option was specified select all the subsystems, ++ * otherwise if 'none', 'name=' and a subsystem name options were ++ * not specified, let's default to 'all' ++ */ ++ if (all_ss || (!one_ss && !opts->none && !opts->name)) ++ for_each_subsys(ss, i) ++ if (!ss->disabled) ++ opts->subsys_mask |= (1 << i); ++ ++ /* ++ * We either have to specify by name or by subsystems. (So all ++ * empty hierarchies must have a name). ++ */ ++ if (!opts->subsys_mask && !opts->name) ++ return -EINVAL; ++ ++ /* ++ * Option noprefix was introduced just for backward compatibility ++ * with the old cpuset, so we allow noprefix only if mounting just ++ * the cpuset subsystem. ++ */ ++ if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask)) ++ return -EINVAL; ++ ++ /* Can't specify "none" and some subsystems */ ++ if (opts->subsys_mask && opts->none) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data) ++{ ++ int ret = 0; ++ struct cgroup_root *root = cgroup_root_from_kf(kf_root); ++ struct cgroup_sb_opts opts; ++ unsigned int added_mask, removed_mask; ++ ++ if (root == &cgrp_dfl_root) { ++ pr_err("remount is not allowed\n"); ++ return -EINVAL; ++ } ++ ++ mutex_lock(&cgroup_mutex); ++ ++ /* See what subsystems are wanted */ ++ ret = parse_cgroupfs_options(data, &opts); ++ if (ret) ++ goto out_unlock; ++ ++ if (opts.subsys_mask != root->subsys_mask || opts.release_agent) ++ pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n", ++ task_tgid_nr(current), current->comm); ++ ++ added_mask = opts.subsys_mask & ~root->subsys_mask; ++ removed_mask = root->subsys_mask & ~opts.subsys_mask; ++ ++ /* Don't allow flags or name to change at remount */ ++ if ((opts.flags ^ root->flags) || ++ (opts.name && strcmp(opts.name, root->name))) { ++ pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n", ++ opts.flags, opts.name ?: "", root->flags, root->name); ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ /* remounting is not allowed for populated hierarchies */ ++ if (!list_empty(&root->cgrp.self.children)) { ++ ret = -EBUSY; ++ goto out_unlock; ++ } ++ ++ ret = rebind_subsystems(root, added_mask); ++ if (ret) ++ goto out_unlock; ++ ++ rebind_subsystems(&cgrp_dfl_root, removed_mask); ++ ++ if (opts.release_agent) { ++ spin_lock(&release_agent_path_lock); ++ strcpy(root->release_agent_path, opts.release_agent); ++ spin_unlock(&release_agent_path_lock); ++ } ++ out_unlock: ++ kfree(opts.release_agent); ++ kfree(opts.name); ++ mutex_unlock(&cgroup_mutex); ++ return ret; ++} ++ ++/* ++ * To reduce the fork() overhead for systems that are not actually using ++ * their cgroups capability, we don't maintain the lists running through ++ * each css_set to its tasks until we see the list actually used - in other ++ * words after the first mount. ++ */ ++static bool use_task_css_set_links __read_mostly; ++ ++static void cgroup_enable_task_cg_lists(void) ++{ ++ struct task_struct *p, *g; ++ ++ down_write(&css_set_rwsem); ++ ++ if (use_task_css_set_links) ++ goto out_unlock; ++ ++ use_task_css_set_links = true; ++ ++ /* ++ * We need tasklist_lock because RCU is not safe against ++ * while_each_thread(). Besides, a forking task that has passed ++ * cgroup_post_fork() without seeing use_task_css_set_links = 1 ++ * is not guaranteed to have its child immediately visible in the ++ * tasklist if we walk through it with RCU. ++ */ ++ read_lock(&tasklist_lock); ++ do_each_thread(g, p) { ++ WARN_ON_ONCE(!list_empty(&p->cg_list) || ++ task_css_set(p) != &init_css_set); ++ ++ /* ++ * We should check if the process is exiting, otherwise ++ * it will race with cgroup_exit() in that the list ++ * entry won't be deleted though the process has exited. ++ * Do it while holding siglock so that we don't end up ++ * racing against cgroup_exit(). ++ */ ++ spin_lock_irq(&p->sighand->siglock); ++ if (!(p->flags & PF_EXITING)) { ++ struct css_set *cset = task_css_set(p); ++ ++ list_add(&p->cg_list, &cset->tasks); ++ get_css_set(cset); ++ } ++ spin_unlock_irq(&p->sighand->siglock); ++ } while_each_thread(g, p); ++ read_unlock(&tasklist_lock); ++out_unlock: ++ up_write(&css_set_rwsem); ++} ++ ++static void init_cgroup_housekeeping(struct cgroup *cgrp) ++{ ++ struct cgroup_subsys *ss; ++ int ssid; ++ ++ INIT_LIST_HEAD(&cgrp->self.sibling); ++ INIT_LIST_HEAD(&cgrp->self.children); ++ INIT_LIST_HEAD(&cgrp->cset_links); ++ INIT_LIST_HEAD(&cgrp->pidlists); ++ mutex_init(&cgrp->pidlist_mutex); ++ cgrp->self.cgroup = cgrp; ++ cgrp->self.flags |= CSS_ONLINE; ++ ++ for_each_subsys(ss, ssid) ++ INIT_LIST_HEAD(&cgrp->e_csets[ssid]); ++ ++ init_waitqueue_head(&cgrp->offline_waitq); ++ INIT_WORK(&cgrp->release_agent_work, cgroup_release_agent); ++} ++ ++static void init_cgroup_root(struct cgroup_root *root, ++ struct cgroup_sb_opts *opts) ++{ ++ struct cgroup *cgrp = &root->cgrp; ++ ++ INIT_LIST_HEAD(&root->root_list); ++ atomic_set(&root->nr_cgrps, 1); ++ cgrp->root = root; ++ init_cgroup_housekeeping(cgrp); ++ idr_init(&root->cgroup_idr); ++ ++ root->flags = opts->flags; ++ if (opts->release_agent) ++ strcpy(root->release_agent_path, opts->release_agent); ++ if (opts->name) ++ strcpy(root->name, opts->name); ++ if (opts->cpuset_clone_children) ++ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); ++} ++ ++static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask) ++{ ++ LIST_HEAD(tmp_links); ++ struct cgroup *root_cgrp = &root->cgrp; ++ struct cftype *base_files; ++ struct css_set *cset; ++ int i, ret; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_NOWAIT); ++ if (ret < 0) ++ goto out; ++ root_cgrp->id = ret; ++ ++ ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 0, ++ GFP_KERNEL); ++ if (ret) ++ goto out; ++ ++ /* ++ * We're accessing css_set_count without locking css_set_rwsem here, ++ * but that's OK - it can only be increased by someone holding ++ * cgroup_lock, and that's us. The worst that can happen is that we ++ * have some link structures left over ++ */ ++ ret = allocate_cgrp_cset_links(css_set_count, &tmp_links); ++ if (ret) ++ goto cancel_ref; ++ ++ ret = cgroup_init_root_id(root); ++ if (ret) ++ goto cancel_ref; ++ ++ root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops, ++ KERNFS_ROOT_CREATE_DEACTIVATED, ++ root_cgrp); ++ if (IS_ERR(root->kf_root)) { ++ ret = PTR_ERR(root->kf_root); ++ goto exit_root_id; ++ } ++ root_cgrp->kn = root->kf_root->kn; ++ ++ if (root == &cgrp_dfl_root) ++ base_files = cgroup_dfl_base_files; ++ else ++ base_files = cgroup_legacy_base_files; ++ ++ ret = cgroup_addrm_files(root_cgrp, base_files, true); ++ if (ret) ++ goto destroy_root; ++ ++ ret = rebind_subsystems(root, ss_mask); ++ if (ret) ++ goto destroy_root; ++ ++ /* ++ * There must be no failure case after here, since rebinding takes ++ * care of subsystems' refcounts, which are explicitly dropped in ++ * the failure exit path. ++ */ ++ list_add(&root->root_list, &cgroup_roots); ++ cgroup_root_count++; ++ ++ /* ++ * Link the root cgroup in this hierarchy into all the css_set ++ * objects. ++ */ ++ down_write(&css_set_rwsem); ++ hash_for_each(css_set_table, i, cset, hlist) ++ link_css_set(&tmp_links, cset, root_cgrp); ++ up_write(&css_set_rwsem); ++ ++ BUG_ON(!list_empty(&root_cgrp->self.children)); ++ BUG_ON(atomic_read(&root->nr_cgrps) != 1); ++ ++ kernfs_activate(root_cgrp->kn); ++ ret = 0; ++ goto out; ++ ++destroy_root: ++ kernfs_destroy_root(root->kf_root); ++ root->kf_root = NULL; ++exit_root_id: ++ cgroup_exit_root_id(root); ++cancel_ref: ++ percpu_ref_exit(&root_cgrp->self.refcnt); ++out: ++ free_cgrp_cset_links(&tmp_links); ++ return ret; ++} ++ ++static struct dentry *cgroup_mount(struct file_system_type *fs_type, ++ int flags, const char *unused_dev_name, ++ void *data) ++{ ++ struct super_block *pinned_sb = NULL; ++ struct cgroup_subsys *ss; ++ struct cgroup_root *root; ++ struct cgroup_sb_opts opts; ++ struct dentry *dentry; ++ int ret; ++ int i; ++ bool new_sb; ++ ++ /* ++ * The first time anyone tries to mount a cgroup, enable the list ++ * linking each css_set to its tasks and fix up all existing tasks. ++ */ ++ if (!use_task_css_set_links) ++ cgroup_enable_task_cg_lists(); ++ ++ mutex_lock(&cgroup_mutex); ++ ++ /* First find the desired set of subsystems */ ++ ret = parse_cgroupfs_options(data, &opts); ++ if (ret) ++ goto out_unlock; ++ ++ /* look for a matching existing root */ ++ if (opts.flags & CGRP_ROOT_SANE_BEHAVIOR) { ++ cgrp_dfl_root_visible = true; ++ root = &cgrp_dfl_root; ++ cgroup_get(&root->cgrp); ++ ret = 0; ++ goto out_unlock; ++ } ++ ++ /* ++ * Destruction of cgroup root is asynchronous, so subsystems may ++ * still be dying after the previous unmount. Let's drain the ++ * dying subsystems. We just need to ensure that the ones ++ * unmounted previously finish dying and don't care about new ones ++ * starting. Testing ref liveliness is good enough. ++ */ ++ for_each_subsys(ss, i) { ++ if (!(opts.subsys_mask & (1 << i)) || ++ ss->root == &cgrp_dfl_root) ++ continue; ++ ++ if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) { ++ mutex_unlock(&cgroup_mutex); ++ msleep(10); ++ ret = restart_syscall(); ++ goto out_free; ++ } ++ cgroup_put(&ss->root->cgrp); ++ } ++ ++ for_each_root(root) { ++ bool name_match = false; ++ ++ if (root == &cgrp_dfl_root) ++ continue; ++ ++ /* ++ * If we asked for a name then it must match. Also, if ++ * name matches but sybsys_mask doesn't, we should fail. ++ * Remember whether name matched. ++ */ ++ if (opts.name) { ++ if (strcmp(opts.name, root->name)) ++ continue; ++ name_match = true; ++ } ++ ++ /* ++ * If we asked for subsystems (or explicitly for no ++ * subsystems) then they must match. ++ */ ++ if ((opts.subsys_mask || opts.none) && ++ (opts.subsys_mask != root->subsys_mask)) { ++ if (!name_match) ++ continue; ++ ret = -EBUSY; ++ goto out_unlock; ++ } ++ ++ if (root->flags ^ opts.flags) ++ pr_warn("new mount options do not match the existing superblock, will be ignored\n"); ++ ++ /* ++ * We want to reuse @root whose lifetime is governed by its ++ * ->cgrp. Let's check whether @root is alive and keep it ++ * that way. As cgroup_kill_sb() can happen anytime, we ++ * want to block it by pinning the sb so that @root doesn't ++ * get killed before mount is complete. ++ * ++ * With the sb pinned, tryget_live can reliably indicate ++ * whether @root can be reused. If it's being killed, ++ * drain it. We can use wait_queue for the wait but this ++ * path is super cold. Let's just sleep a bit and retry. ++ */ ++ pinned_sb = kernfs_pin_sb(root->kf_root, NULL); ++ if (IS_ERR(pinned_sb) || ++ !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) { ++ mutex_unlock(&cgroup_mutex); ++ if (!IS_ERR_OR_NULL(pinned_sb)) ++ deactivate_super(pinned_sb); ++ msleep(10); ++ ret = restart_syscall(); ++ goto out_free; ++ } ++ ++ ret = 0; ++ goto out_unlock; ++ } ++ ++ /* ++ * No such thing, create a new one. name= matching without subsys ++ * specification is allowed for already existing hierarchies but we ++ * can't create new one without subsys specification. ++ */ ++ if (!opts.subsys_mask && !opts.none) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ root = kzalloc(sizeof(*root), GFP_KERNEL); ++ if (!root) { ++ ret = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ init_cgroup_root(root, &opts); ++ ++ ret = cgroup_setup_root(root, opts.subsys_mask); ++ if (ret) ++ cgroup_free_root(root); ++ ++out_unlock: ++ mutex_unlock(&cgroup_mutex); ++out_free: ++ kfree(opts.release_agent); ++ kfree(opts.name); ++ ++ if (ret) ++ return ERR_PTR(ret); ++ ++ dentry = kernfs_mount(fs_type, flags, root->kf_root, ++ CGROUP_SUPER_MAGIC, &new_sb); ++ if (IS_ERR(dentry) || !new_sb) ++ cgroup_put(&root->cgrp); ++ ++ /* ++ * If @pinned_sb, we're reusing an existing root and holding an ++ * extra ref on its sb. Mount is complete. Put the extra ref. ++ */ ++ if (pinned_sb) { ++ WARN_ON(new_sb); ++ deactivate_super(pinned_sb); ++ } ++ ++ return dentry; ++} ++ ++static void cgroup_kill_sb(struct super_block *sb) ++{ ++ struct kernfs_root *kf_root = kernfs_root_from_sb(sb); ++ struct cgroup_root *root = cgroup_root_from_kf(kf_root); ++ ++ /* ++ * If @root doesn't have any mounts or children, start killing it. ++ * This prevents new mounts by disabling percpu_ref_tryget_live(). ++ * cgroup_mount() may wait for @root's release. ++ * ++ * And don't kill the default root. ++ */ ++ if (!list_empty(&root->cgrp.self.children) || ++ root == &cgrp_dfl_root) ++ cgroup_put(&root->cgrp); ++ else ++ percpu_ref_kill(&root->cgrp.self.refcnt); ++ ++ kernfs_kill_sb(sb); ++} ++ ++static struct file_system_type cgroup_fs_type = { ++ .name = "cgroup", ++ .mount = cgroup_mount, ++ .kill_sb = cgroup_kill_sb, ++}; ++ ++/** ++ * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy ++ * @task: target task ++ * @buf: the buffer to write the path into ++ * @buflen: the length of the buffer ++ * ++ * Determine @task's cgroup on the first (the one with the lowest non-zero ++ * hierarchy_id) cgroup hierarchy and copy its path into @buf. This ++ * function grabs cgroup_mutex and shouldn't be used inside locks used by ++ * cgroup controller callbacks. ++ * ++ * Return value is the same as kernfs_path(). ++ */ ++char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen) ++{ ++ struct cgroup_root *root; ++ struct cgroup *cgrp; ++ int hierarchy_id = 1; ++ char *path = NULL; ++ ++ mutex_lock(&cgroup_mutex); ++ down_read(&css_set_rwsem); ++ ++ root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id); ++ ++ if (root) { ++ cgrp = task_cgroup_from_root(task, root); ++ path = cgroup_path(cgrp, buf, buflen); ++ } else { ++ /* if no hierarchy exists, everyone is in "/" */ ++ if (strlcpy(buf, "/", buflen) < buflen) ++ path = buf; ++ } ++ ++ up_read(&css_set_rwsem); ++ mutex_unlock(&cgroup_mutex); ++ return path; ++} ++EXPORT_SYMBOL_GPL(task_cgroup_path); ++ ++/* used to track tasks and other necessary states during migration */ ++struct cgroup_taskset { ++ /* the src and dst cset list running through cset->mg_node */ ++ struct list_head src_csets; ++ struct list_head dst_csets; ++ ++ /* ++ * Fields for cgroup_taskset_*() iteration. ++ * ++ * Before migration is committed, the target migration tasks are on ++ * ->mg_tasks of the csets on ->src_csets. After, on ->mg_tasks of ++ * the csets on ->dst_csets. ->csets point to either ->src_csets ++ * or ->dst_csets depending on whether migration is committed. ++ * ++ * ->cur_csets and ->cur_task point to the current task position ++ * during iteration. ++ */ ++ struct list_head *csets; ++ struct css_set *cur_cset; ++ struct task_struct *cur_task; ++}; ++ ++/** ++ * cgroup_taskset_first - reset taskset and return the first task ++ * @tset: taskset of interest ++ * ++ * @tset iteration is initialized and the first task is returned. ++ */ ++struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset) ++{ ++ tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node); ++ tset->cur_task = NULL; ++ ++ return cgroup_taskset_next(tset); ++} ++ ++/** ++ * cgroup_taskset_next - iterate to the next task in taskset ++ * @tset: taskset of interest ++ * ++ * Return the next task in @tset. Iteration must have been initialized ++ * with cgroup_taskset_first(). ++ */ ++struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset) ++{ ++ struct css_set *cset = tset->cur_cset; ++ struct task_struct *task = tset->cur_task; ++ ++ while (&cset->mg_node != tset->csets) { ++ if (!task) ++ task = list_first_entry(&cset->mg_tasks, ++ struct task_struct, cg_list); ++ else ++ task = list_next_entry(task, cg_list); ++ ++ if (&task->cg_list != &cset->mg_tasks) { ++ tset->cur_cset = cset; ++ tset->cur_task = task; ++ return task; ++ } ++ ++ cset = list_next_entry(cset, mg_node); ++ task = NULL; ++ } ++ ++ return NULL; ++} ++ ++/** ++ * cgroup_task_migrate - move a task from one cgroup to another. ++ * @old_cgrp: the cgroup @tsk is being migrated from ++ * @tsk: the task being migrated ++ * @new_cset: the new css_set @tsk is being attached to ++ * ++ * Must be called with cgroup_mutex, threadgroup and css_set_rwsem locked. ++ */ ++static void cgroup_task_migrate(struct cgroup *old_cgrp, ++ struct task_struct *tsk, ++ struct css_set *new_cset) ++{ ++ struct css_set *old_cset; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ lockdep_assert_held(&css_set_rwsem); ++ ++ /* ++ * We are synchronized through threadgroup_lock() against PF_EXITING ++ * setting such that we can't race against cgroup_exit() changing the ++ * css_set to init_css_set and dropping the old one. ++ */ ++ WARN_ON_ONCE(tsk->flags & PF_EXITING); ++ old_cset = task_css_set(tsk); ++ ++ get_css_set(new_cset); ++ rcu_assign_pointer(tsk->cgroups, new_cset); ++ ++ /* ++ * Use move_tail so that cgroup_taskset_first() still returns the ++ * leader after migration. This works because cgroup_migrate() ++ * ensures that the dst_cset of the leader is the first on the ++ * tset's dst_csets list. ++ */ ++ list_move_tail(&tsk->cg_list, &new_cset->mg_tasks); ++ ++ /* ++ * We just gained a reference on old_cset by taking it from the ++ * task. As trading it for new_cset is protected by cgroup_mutex, ++ * we're safe to drop it here; it will be freed under RCU. ++ */ ++ put_css_set_locked(old_cset); ++} ++ ++/** ++ * cgroup_migrate_finish - cleanup after attach ++ * @preloaded_csets: list of preloaded css_sets ++ * ++ * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See ++ * those functions for details. ++ */ ++static void cgroup_migrate_finish(struct list_head *preloaded_csets) ++{ ++ struct css_set *cset, *tmp_cset; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ down_write(&css_set_rwsem); ++ list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) { ++ cset->mg_src_cgrp = NULL; ++ cset->mg_dst_cset = NULL; ++ list_del_init(&cset->mg_preload_node); ++ put_css_set_locked(cset); ++ } ++ up_write(&css_set_rwsem); ++} ++ ++/** ++ * cgroup_migrate_add_src - add a migration source css_set ++ * @src_cset: the source css_set to add ++ * @dst_cgrp: the destination cgroup ++ * @preloaded_csets: list of preloaded css_sets ++ * ++ * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin ++ * @src_cset and add it to @preloaded_csets, which should later be cleaned ++ * up by cgroup_migrate_finish(). ++ * ++ * This function may be called without holding threadgroup_lock even if the ++ * target is a process. Threads may be created and destroyed but as long ++ * as cgroup_mutex is not dropped, no new css_set can be put into play and ++ * the preloaded css_sets are guaranteed to cover all migrations. ++ */ ++static void cgroup_migrate_add_src(struct css_set *src_cset, ++ struct cgroup *dst_cgrp, ++ struct list_head *preloaded_csets) ++{ ++ struct cgroup *src_cgrp; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ lockdep_assert_held(&css_set_rwsem); ++ ++ src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root); ++ ++ if (!list_empty(&src_cset->mg_preload_node)) ++ return; ++ ++ WARN_ON(src_cset->mg_src_cgrp); ++ WARN_ON(!list_empty(&src_cset->mg_tasks)); ++ WARN_ON(!list_empty(&src_cset->mg_node)); ++ ++ src_cset->mg_src_cgrp = src_cgrp; ++ get_css_set(src_cset); ++ list_add(&src_cset->mg_preload_node, preloaded_csets); ++} ++ ++/** ++ * cgroup_migrate_prepare_dst - prepare destination css_sets for migration ++ * @dst_cgrp: the destination cgroup (may be %NULL) ++ * @preloaded_csets: list of preloaded source css_sets ++ * ++ * Tasks are about to be moved to @dst_cgrp and all the source css_sets ++ * have been preloaded to @preloaded_csets. This function looks up and ++ * pins all destination css_sets, links each to its source, and append them ++ * to @preloaded_csets. If @dst_cgrp is %NULL, the destination of each ++ * source css_set is assumed to be its cgroup on the default hierarchy. ++ * ++ * This function must be called after cgroup_migrate_add_src() has been ++ * called on each migration source css_set. After migration is performed ++ * using cgroup_migrate(), cgroup_migrate_finish() must be called on ++ * @preloaded_csets. ++ */ ++static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp, ++ struct list_head *preloaded_csets) ++{ ++ LIST_HEAD(csets); ++ struct css_set *src_cset, *tmp_cset; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* ++ * Except for the root, child_subsys_mask must be zero for a cgroup ++ * with tasks so that child cgroups don't compete against tasks. ++ */ ++ if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) && ++ dst_cgrp->child_subsys_mask) ++ return -EBUSY; ++ ++ /* look up the dst cset for each src cset and link it to src */ ++ list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) { ++ struct css_set *dst_cset; ++ ++ dst_cset = find_css_set(src_cset, ++ dst_cgrp ?: src_cset->dfl_cgrp); ++ if (!dst_cset) ++ goto err; ++ ++ WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset); ++ ++ /* ++ * If src cset equals dst, it's noop. Drop the src. ++ * cgroup_migrate() will skip the cset too. Note that we ++ * can't handle src == dst as some nodes are used by both. ++ */ ++ if (src_cset == dst_cset) { ++ src_cset->mg_src_cgrp = NULL; ++ list_del_init(&src_cset->mg_preload_node); ++ put_css_set(src_cset); ++ put_css_set(dst_cset); ++ continue; ++ } ++ ++ src_cset->mg_dst_cset = dst_cset; ++ ++ if (list_empty(&dst_cset->mg_preload_node)) ++ list_add(&dst_cset->mg_preload_node, &csets); ++ else ++ put_css_set(dst_cset); ++ } ++ ++ list_splice_tail(&csets, preloaded_csets); ++ return 0; ++err: ++ cgroup_migrate_finish(&csets); ++ return -ENOMEM; ++} ++ ++/** ++ * cgroup_migrate - migrate a process or task to a cgroup ++ * @cgrp: the destination cgroup ++ * @leader: the leader of the process or the task to migrate ++ * @threadgroup: whether @leader points to the whole process or a single task ++ * ++ * Migrate a process or task denoted by @leader to @cgrp. If migrating a ++ * process, the caller must be holding threadgroup_lock of @leader. The ++ * caller is also responsible for invoking cgroup_migrate_add_src() and ++ * cgroup_migrate_prepare_dst() on the targets before invoking this ++ * function and following up with cgroup_migrate_finish(). ++ * ++ * As long as a controller's ->can_attach() doesn't fail, this function is ++ * guaranteed to succeed. This means that, excluding ->can_attach() ++ * failure, when migrating multiple targets, the success or failure can be ++ * decided for all targets by invoking group_migrate_prepare_dst() before ++ * actually starting migrating. ++ */ ++static int cgroup_migrate(struct cgroup *cgrp, struct task_struct *leader, ++ bool threadgroup) ++{ ++ struct cgroup_taskset tset = { ++ .src_csets = LIST_HEAD_INIT(tset.src_csets), ++ .dst_csets = LIST_HEAD_INIT(tset.dst_csets), ++ .csets = &tset.src_csets, ++ }; ++ struct cgroup_subsys_state *css, *failed_css = NULL; ++ struct css_set *cset, *tmp_cset; ++ struct task_struct *task, *tmp_task; ++ int i, ret; ++ ++ /* ++ * Prevent freeing of tasks while we take a snapshot. Tasks that are ++ * already PF_EXITING could be freed from underneath us unless we ++ * take an rcu_read_lock. ++ */ ++ down_write(&css_set_rwsem); ++ rcu_read_lock(); ++ task = leader; ++ do { ++ /* @task either already exited or can't exit until the end */ ++ if (task->flags & PF_EXITING) ++ goto next; ++ ++ /* leave @task alone if post_fork() hasn't linked it yet */ ++ if (list_empty(&task->cg_list)) ++ goto next; ++ ++ cset = task_css_set(task); ++ if (!cset->mg_src_cgrp) ++ goto next; ++ ++ /* ++ * cgroup_taskset_first() must always return the leader. ++ * Take care to avoid disturbing the ordering. ++ */ ++ list_move_tail(&task->cg_list, &cset->mg_tasks); ++ if (list_empty(&cset->mg_node)) ++ list_add_tail(&cset->mg_node, &tset.src_csets); ++ if (list_empty(&cset->mg_dst_cset->mg_node)) ++ list_move_tail(&cset->mg_dst_cset->mg_node, ++ &tset.dst_csets); ++ next: ++ if (!threadgroup) ++ break; ++ } while_each_thread(leader, task); ++ rcu_read_unlock(); ++ up_write(&css_set_rwsem); ++ ++ /* methods shouldn't be called if no task is actually migrating */ ++ if (list_empty(&tset.src_csets)) ++ return 0; ++ ++ /* check that we can legitimately attach to the cgroup */ ++ for_each_e_css(css, i, cgrp) { ++ if (css->ss->can_attach) { ++ ret = css->ss->can_attach(css, &tset); ++ if (ret) { ++ failed_css = css; ++ goto out_cancel_attach; ++ } ++ } ++ } ++ ++ /* ++ * Now that we're guaranteed success, proceed to move all tasks to ++ * the new cgroup. There are no failure cases after here, so this ++ * is the commit point. ++ */ ++ down_write(&css_set_rwsem); ++ list_for_each_entry(cset, &tset.src_csets, mg_node) { ++ list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) ++ cgroup_task_migrate(cset->mg_src_cgrp, task, ++ cset->mg_dst_cset); ++ } ++ up_write(&css_set_rwsem); ++ ++ /* ++ * Migration is committed, all target tasks are now on dst_csets. ++ * Nothing is sensitive to fork() after this point. Notify ++ * controllers that migration is complete. ++ */ ++ tset.csets = &tset.dst_csets; ++ ++ for_each_e_css(css, i, cgrp) ++ if (css->ss->attach) ++ css->ss->attach(css, &tset); ++ ++ ret = 0; ++ goto out_release_tset; ++ ++out_cancel_attach: ++ for_each_e_css(css, i, cgrp) { ++ if (css == failed_css) ++ break; ++ if (css->ss->cancel_attach) ++ css->ss->cancel_attach(css, &tset); ++ } ++out_release_tset: ++ down_write(&css_set_rwsem); ++ list_splice_init(&tset.dst_csets, &tset.src_csets); ++ list_for_each_entry_safe(cset, tmp_cset, &tset.src_csets, mg_node) { ++ list_splice_tail_init(&cset->mg_tasks, &cset->tasks); ++ list_del_init(&cset->mg_node); ++ } ++ up_write(&css_set_rwsem); ++ return ret; ++} ++ ++/** ++ * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup ++ * @dst_cgrp: the cgroup to attach to ++ * @leader: the task or the leader of the threadgroup to be attached ++ * @threadgroup: attach the whole threadgroup? ++ * ++ * Call holding cgroup_mutex and threadgroup_lock of @leader. ++ */ ++static int cgroup_attach_task(struct cgroup *dst_cgrp, ++ struct task_struct *leader, bool threadgroup) ++{ ++ LIST_HEAD(preloaded_csets); ++ struct task_struct *task; ++ int ret; ++ ++ /* look up all src csets */ ++ down_read(&css_set_rwsem); ++ rcu_read_lock(); ++ task = leader; ++ do { ++ cgroup_migrate_add_src(task_css_set(task), dst_cgrp, ++ &preloaded_csets); ++ if (!threadgroup) ++ break; ++ } while_each_thread(leader, task); ++ rcu_read_unlock(); ++ up_read(&css_set_rwsem); ++ ++ /* prepare dst csets and commit */ ++ ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets); ++ if (!ret) ++ ret = cgroup_migrate(dst_cgrp, leader, threadgroup); ++ ++ cgroup_migrate_finish(&preloaded_csets); ++ return ret; ++} ++ ++/* ++ * Find the task_struct of the task to attach by vpid and pass it along to the ++ * function to attach either it or all tasks in its threadgroup. Will lock ++ * cgroup_mutex and threadgroup. ++ */ ++static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, ++ size_t nbytes, loff_t off, bool threadgroup) ++{ ++ struct task_struct *tsk; ++ const struct cred *cred = current_cred(), *tcred; ++ struct cgroup *cgrp; ++ pid_t pid; ++ int ret; ++ ++ if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) ++ return -EINVAL; ++ ++ cgrp = cgroup_kn_lock_live(of->kn); ++ if (!cgrp) ++ return -ENODEV; ++ ++retry_find_task: ++ rcu_read_lock(); ++ if (pid) { ++ tsk = find_task_by_vpid(pid); ++ if (!tsk) { ++ rcu_read_unlock(); ++ ret = -ESRCH; ++ goto out_unlock_cgroup; ++ } ++ /* ++ * even if we're attaching all tasks in the thread group, we ++ * only need to check permissions on one of them. ++ */ ++ tcred = __task_cred(tsk); ++ if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) && ++ !uid_eq(cred->euid, tcred->uid) && ++ !uid_eq(cred->euid, tcred->suid)) { ++ rcu_read_unlock(); ++ ret = -EACCES; ++ goto out_unlock_cgroup; ++ } ++ } else ++ tsk = current; ++ ++ if (threadgroup) ++ tsk = tsk->group_leader; ++ ++ /* ++ * Workqueue threads may acquire PF_NO_SETAFFINITY and become ++ * trapped in a cpuset, or RT worker may be born in a cgroup ++ * with no rt_runtime allocated. Just say no. ++ */ ++ if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) { ++ ret = -EINVAL; ++ rcu_read_unlock(); ++ goto out_unlock_cgroup; ++ } ++ ++ get_task_struct(tsk); ++ rcu_read_unlock(); ++ ++ threadgroup_lock(tsk); ++ if (threadgroup) { ++ if (!thread_group_leader(tsk)) { ++ /* ++ * a race with de_thread from another thread's exec() ++ * may strip us of our leadership, if this happens, ++ * there is no choice but to throw this task away and ++ * try again; this is ++ * "double-double-toil-and-trouble-check locking". ++ */ ++ threadgroup_unlock(tsk); ++ put_task_struct(tsk); ++ goto retry_find_task; ++ } ++ } ++ ++ ret = cgroup_attach_task(cgrp, tsk, threadgroup); ++ ++ threadgroup_unlock(tsk); ++ ++ put_task_struct(tsk); ++out_unlock_cgroup: ++ cgroup_kn_unlock(of->kn); ++ return ret ?: nbytes; ++} ++ ++/** ++ * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from' ++ * @from: attach to all cgroups of a given task ++ * @tsk: the task to be attached ++ */ ++int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk) ++{ ++ struct cgroup_root *root; ++ int retval = 0; ++ ++ mutex_lock(&cgroup_mutex); ++ for_each_root(root) { ++ struct cgroup *from_cgrp; ++ ++ if (root == &cgrp_dfl_root) ++ continue; ++ ++ down_read(&css_set_rwsem); ++ from_cgrp = task_cgroup_from_root(from, root); ++ up_read(&css_set_rwsem); ++ ++ retval = cgroup_attach_task(from_cgrp, tsk, false); ++ if (retval) ++ break; ++ } ++ mutex_unlock(&cgroup_mutex); ++ ++ return retval; ++} ++EXPORT_SYMBOL_GPL(cgroup_attach_task_all); ++ ++static ssize_t cgroup_tasks_write(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, loff_t off) ++{ ++ return __cgroup_procs_write(of, buf, nbytes, off, false); ++} ++ ++static ssize_t cgroup_procs_write(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, loff_t off) ++{ ++ return __cgroup_procs_write(of, buf, nbytes, off, true); ++} ++ ++static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, loff_t off) ++{ ++ struct cgroup *cgrp; ++ ++ BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX); ++ ++ cgrp = cgroup_kn_lock_live(of->kn); ++ if (!cgrp) ++ return -ENODEV; ++ spin_lock(&release_agent_path_lock); ++ strlcpy(cgrp->root->release_agent_path, strstrip(buf), ++ sizeof(cgrp->root->release_agent_path)); ++ spin_unlock(&release_agent_path_lock); ++ cgroup_kn_unlock(of->kn); ++ return nbytes; ++} ++ ++static int cgroup_release_agent_show(struct seq_file *seq, void *v) ++{ ++ struct cgroup *cgrp = seq_css(seq)->cgroup; ++ ++ spin_lock(&release_agent_path_lock); ++ seq_puts(seq, cgrp->root->release_agent_path); ++ spin_unlock(&release_agent_path_lock); ++ seq_putc(seq, '\n'); ++ return 0; ++} ++ ++static int cgroup_sane_behavior_show(struct seq_file *seq, void *v) ++{ ++ seq_puts(seq, "0\n"); ++ return 0; ++} ++ ++static void cgroup_print_ss_mask(struct seq_file *seq, unsigned int ss_mask) ++{ ++ struct cgroup_subsys *ss; ++ bool printed = false; ++ int ssid; ++ ++ for_each_subsys(ss, ssid) { ++ if (ss_mask & (1 << ssid)) { ++ if (printed) ++ seq_putc(seq, ' '); ++ seq_printf(seq, "%s", ss->name); ++ printed = true; ++ } ++ } ++ if (printed) ++ seq_putc(seq, '\n'); ++} ++ ++/* show controllers which are currently attached to the default hierarchy */ ++static int cgroup_root_controllers_show(struct seq_file *seq, void *v) ++{ ++ struct cgroup *cgrp = seq_css(seq)->cgroup; ++ ++ cgroup_print_ss_mask(seq, cgrp->root->subsys_mask & ++ ~cgrp_dfl_root_inhibit_ss_mask); ++ return 0; ++} ++ ++/* show controllers which are enabled from the parent */ ++static int cgroup_controllers_show(struct seq_file *seq, void *v) ++{ ++ struct cgroup *cgrp = seq_css(seq)->cgroup; ++ ++ cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->subtree_control); ++ return 0; ++} ++ ++/* show controllers which are enabled for a given cgroup's children */ ++static int cgroup_subtree_control_show(struct seq_file *seq, void *v) ++{ ++ struct cgroup *cgrp = seq_css(seq)->cgroup; ++ ++ cgroup_print_ss_mask(seq, cgrp->subtree_control); ++ return 0; ++} ++ ++/** ++ * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy ++ * @cgrp: root of the subtree to update csses for ++ * ++ * @cgrp's child_subsys_mask has changed and its subtree's (self excluded) ++ * css associations need to be updated accordingly. This function looks up ++ * all css_sets which are attached to the subtree, creates the matching ++ * updated css_sets and migrates the tasks to the new ones. ++ */ ++static int cgroup_update_dfl_csses(struct cgroup *cgrp) ++{ ++ LIST_HEAD(preloaded_csets); ++ struct cgroup_subsys_state *css; ++ struct css_set *src_cset; ++ int ret; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* look up all csses currently attached to @cgrp's subtree */ ++ down_read(&css_set_rwsem); ++ css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) { ++ struct cgrp_cset_link *link; ++ ++ /* self is not affected by child_subsys_mask change */ ++ if (css->cgroup == cgrp) ++ continue; ++ ++ list_for_each_entry(link, &css->cgroup->cset_links, cset_link) ++ cgroup_migrate_add_src(link->cset, cgrp, ++ &preloaded_csets); ++ } ++ up_read(&css_set_rwsem); ++ ++ /* NULL dst indicates self on default hierarchy */ ++ ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets); ++ if (ret) ++ goto out_finish; ++ ++ list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) { ++ struct task_struct *last_task = NULL, *task; ++ ++ /* src_csets precede dst_csets, break on the first dst_cset */ ++ if (!src_cset->mg_src_cgrp) ++ break; ++ ++ /* ++ * All tasks in src_cset need to be migrated to the ++ * matching dst_cset. Empty it process by process. We ++ * walk tasks but migrate processes. The leader might even ++ * belong to a different cset but such src_cset would also ++ * be among the target src_csets because the default ++ * hierarchy enforces per-process membership. ++ */ ++ while (true) { ++ down_read(&css_set_rwsem); ++ task = list_first_entry_or_null(&src_cset->tasks, ++ struct task_struct, cg_list); ++ if (task) { ++ task = task->group_leader; ++ WARN_ON_ONCE(!task_css_set(task)->mg_src_cgrp); ++ get_task_struct(task); ++ } ++ up_read(&css_set_rwsem); ++ ++ if (!task) ++ break; ++ ++ /* guard against possible infinite loop */ ++ if (WARN(last_task == task, ++ "cgroup: update_dfl_csses failed to make progress, aborting in inconsistent state\n")) ++ goto out_finish; ++ last_task = task; ++ ++ threadgroup_lock(task); ++ /* raced against de_thread() from another thread? */ ++ if (!thread_group_leader(task)) { ++ threadgroup_unlock(task); ++ put_task_struct(task); ++ continue; ++ } ++ ++ ret = cgroup_migrate(src_cset->dfl_cgrp, task, true); ++ ++ threadgroup_unlock(task); ++ put_task_struct(task); ++ ++ if (WARN(ret, "cgroup: failed to update controllers for the default hierarchy (%d), further operations may crash or hang\n", ret)) ++ goto out_finish; ++ } ++ } ++ ++out_finish: ++ cgroup_migrate_finish(&preloaded_csets); ++ return ret; ++} ++ ++/* change the enabled child controllers for a cgroup in the default hierarchy */ ++static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, ++ loff_t off) ++{ ++ unsigned int enable = 0, disable = 0; ++ unsigned int css_enable, css_disable, old_sc, new_sc, old_ss, new_ss; ++ struct cgroup *cgrp, *child; ++ struct cgroup_subsys *ss; ++ char *tok; ++ int ssid, ret; ++ ++ /* ++ * Parse input - space separated list of subsystem names prefixed ++ * with either + or -. ++ */ ++ buf = strstrip(buf); ++ while ((tok = strsep(&buf, " "))) { ++ if (tok[0] == '\0') ++ continue; ++ for_each_subsys(ss, ssid) { ++ if (ss->disabled || strcmp(tok + 1, ss->name) || ++ ((1 << ss->id) & cgrp_dfl_root_inhibit_ss_mask)) ++ continue; ++ ++ if (*tok == '+') { ++ enable |= 1 << ssid; ++ disable &= ~(1 << ssid); ++ } else if (*tok == '-') { ++ disable |= 1 << ssid; ++ enable &= ~(1 << ssid); ++ } else { ++ return -EINVAL; ++ } ++ break; ++ } ++ if (ssid == CGROUP_SUBSYS_COUNT) ++ return -EINVAL; ++ } ++ ++ cgrp = cgroup_kn_lock_live(of->kn); ++ if (!cgrp) ++ return -ENODEV; ++ ++ for_each_subsys(ss, ssid) { ++ if (enable & (1 << ssid)) { ++ if (cgrp->subtree_control & (1 << ssid)) { ++ enable &= ~(1 << ssid); ++ continue; ++ } ++ ++ /* unavailable or not enabled on the parent? */ ++ if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) || ++ (cgroup_parent(cgrp) && ++ !(cgroup_parent(cgrp)->subtree_control & (1 << ssid)))) { ++ ret = -ENOENT; ++ goto out_unlock; ++ } ++ } else if (disable & (1 << ssid)) { ++ if (!(cgrp->subtree_control & (1 << ssid))) { ++ disable &= ~(1 << ssid); ++ continue; ++ } ++ ++ /* a child has it enabled? */ ++ cgroup_for_each_live_child(child, cgrp) { ++ if (child->subtree_control & (1 << ssid)) { ++ ret = -EBUSY; ++ goto out_unlock; ++ } ++ } ++ } ++ } ++ ++ if (!enable && !disable) { ++ ret = 0; ++ goto out_unlock; ++ } ++ ++ /* ++ * Except for the root, subtree_control must be zero for a cgroup ++ * with tasks so that child cgroups don't compete against tasks. ++ */ ++ if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) { ++ ret = -EBUSY; ++ goto out_unlock; ++ } ++ ++ /* ++ * Update subsys masks and calculate what needs to be done. More ++ * subsystems than specified may need to be enabled or disabled ++ * depending on subsystem dependencies. ++ */ ++ old_sc = cgrp->subtree_control; ++ old_ss = cgrp->child_subsys_mask; ++ new_sc = (old_sc | enable) & ~disable; ++ new_ss = cgroup_calc_child_subsys_mask(cgrp, new_sc); ++ ++ css_enable = ~old_ss & new_ss; ++ css_disable = old_ss & ~new_ss; ++ enable |= css_enable; ++ disable |= css_disable; ++ ++ /* ++ * Because css offlining is asynchronous, userland might try to ++ * re-enable the same controller while the previous instance is ++ * still around. In such cases, wait till it's gone using ++ * offline_waitq. ++ */ ++ for_each_subsys(ss, ssid) { ++ if (!(css_enable & (1 << ssid))) ++ continue; ++ ++ cgroup_for_each_live_child(child, cgrp) { ++ DEFINE_WAIT(wait); ++ ++ if (!cgroup_css(child, ss)) ++ continue; ++ ++ cgroup_get(child); ++ prepare_to_wait(&child->offline_waitq, &wait, ++ TASK_UNINTERRUPTIBLE); ++ cgroup_kn_unlock(of->kn); ++ schedule(); ++ finish_wait(&child->offline_waitq, &wait); ++ cgroup_put(child); ++ ++ return restart_syscall(); ++ } ++ } ++ ++ cgrp->subtree_control = new_sc; ++ cgrp->child_subsys_mask = new_ss; ++ ++ /* ++ * Create new csses or make the existing ones visible. A css is ++ * created invisible if it's being implicitly enabled through ++ * dependency. An invisible css is made visible when the userland ++ * explicitly enables it. ++ */ ++ for_each_subsys(ss, ssid) { ++ if (!(enable & (1 << ssid))) ++ continue; ++ ++ cgroup_for_each_live_child(child, cgrp) { ++ if (css_enable & (1 << ssid)) ++ ret = create_css(child, ss, ++ cgrp->subtree_control & (1 << ssid)); ++ else ++ ret = cgroup_populate_dir(child, 1 << ssid); ++ if (ret) ++ goto err_undo_css; ++ } ++ } ++ ++ /* ++ * At this point, cgroup_e_css() results reflect the new csses ++ * making the following cgroup_update_dfl_csses() properly update ++ * css associations of all tasks in the subtree. ++ */ ++ ret = cgroup_update_dfl_csses(cgrp); ++ if (ret) ++ goto err_undo_css; ++ ++ /* ++ * All tasks are migrated out of disabled csses. Kill or hide ++ * them. A css is hidden when the userland requests it to be ++ * disabled while other subsystems are still depending on it. The ++ * css must not actively control resources and be in the vanilla ++ * state if it's made visible again later. Controllers which may ++ * be depended upon should provide ->css_reset() for this purpose. ++ */ ++ for_each_subsys(ss, ssid) { ++ if (!(disable & (1 << ssid))) ++ continue; ++ ++ cgroup_for_each_live_child(child, cgrp) { ++ struct cgroup_subsys_state *css = cgroup_css(child, ss); ++ ++ if (css_disable & (1 << ssid)) { ++ kill_css(css); ++ } else { ++ cgroup_clear_dir(child, 1 << ssid); ++ if (ss->css_reset) ++ ss->css_reset(css); ++ } ++ } ++ } ++ ++ /* ++ * The effective csses of all the descendants (excluding @cgrp) may ++ * have changed. Subsystems can optionally subscribe to this event ++ * by implementing ->css_e_css_changed() which is invoked if any of ++ * the effective csses seen from the css's cgroup may have changed. ++ */ ++ for_each_subsys(ss, ssid) { ++ struct cgroup_subsys_state *this_css = cgroup_css(cgrp, ss); ++ struct cgroup_subsys_state *css; ++ ++ if (!ss->css_e_css_changed || !this_css) ++ continue; ++ ++ css_for_each_descendant_pre(css, this_css) ++ if (css != this_css) ++ ss->css_e_css_changed(css); ++ } ++ ++ kernfs_activate(cgrp->kn); ++ ret = 0; ++out_unlock: ++ cgroup_kn_unlock(of->kn); ++ return ret ?: nbytes; ++ ++err_undo_css: ++ cgrp->subtree_control = old_sc; ++ cgrp->child_subsys_mask = old_ss; ++ ++ for_each_subsys(ss, ssid) { ++ if (!(enable & (1 << ssid))) ++ continue; ++ ++ cgroup_for_each_live_child(child, cgrp) { ++ struct cgroup_subsys_state *css = cgroup_css(child, ss); ++ ++ if (!css) ++ continue; ++ ++ if (css_enable & (1 << ssid)) ++ kill_css(css); ++ else ++ cgroup_clear_dir(child, 1 << ssid); ++ } ++ } ++ goto out_unlock; ++} ++ ++static int cgroup_populated_show(struct seq_file *seq, void *v) ++{ ++ seq_printf(seq, "%d\n", (bool)seq_css(seq)->cgroup->populated_cnt); ++ return 0; ++} ++ ++static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf, ++ size_t nbytes, loff_t off) ++{ ++ struct cgroup *cgrp = of->kn->parent->priv; ++ struct cftype *cft = of->kn->priv; ++ struct cgroup_subsys_state *css; ++ int ret; ++ ++ if (cft->write) ++ return cft->write(of, buf, nbytes, off); ++ ++ /* ++ * kernfs guarantees that a file isn't deleted with operations in ++ * flight, which means that the matching css is and stays alive and ++ * doesn't need to be pinned. The RCU locking is not necessary ++ * either. It's just for the convenience of using cgroup_css(). ++ */ ++ rcu_read_lock(); ++ css = cgroup_css(cgrp, cft->ss); ++ rcu_read_unlock(); ++ ++ if (cft->write_u64) { ++ unsigned long long v; ++ ret = kstrtoull(buf, 0, &v); ++ if (!ret) ++ ret = cft->write_u64(css, cft, v); ++ } else if (cft->write_s64) { ++ long long v; ++ ret = kstrtoll(buf, 0, &v); ++ if (!ret) ++ ret = cft->write_s64(css, cft, v); ++ } else { ++ ret = -EINVAL; ++ } ++ ++ return ret ?: nbytes; ++} ++ ++static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos) ++{ ++ return seq_cft(seq)->seq_start(seq, ppos); ++} ++ ++static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos) ++{ ++ return seq_cft(seq)->seq_next(seq, v, ppos); ++} ++ ++static void cgroup_seqfile_stop(struct seq_file *seq, void *v) ++{ ++ seq_cft(seq)->seq_stop(seq, v); ++} ++ ++static int cgroup_seqfile_show(struct seq_file *m, void *arg) ++{ ++ struct cftype *cft = seq_cft(m); ++ struct cgroup_subsys_state *css = seq_css(m); ++ ++ if (cft->seq_show) ++ return cft->seq_show(m, arg); ++ ++ if (cft->read_u64) ++ seq_printf(m, "%llu\n", cft->read_u64(css, cft)); ++ else if (cft->read_s64) ++ seq_printf(m, "%lld\n", cft->read_s64(css, cft)); ++ else ++ return -EINVAL; ++ return 0; ++} ++ ++static struct kernfs_ops cgroup_kf_single_ops = { ++ .atomic_write_len = PAGE_SIZE, ++ .write = cgroup_file_write, ++ .seq_show = cgroup_seqfile_show, ++}; ++ ++static struct kernfs_ops cgroup_kf_ops = { ++ .atomic_write_len = PAGE_SIZE, ++ .write = cgroup_file_write, ++ .seq_start = cgroup_seqfile_start, ++ .seq_next = cgroup_seqfile_next, ++ .seq_stop = cgroup_seqfile_stop, ++ .seq_show = cgroup_seqfile_show, ++}; ++ ++/* ++ * cgroup_rename - Only allow simple rename of directories in place. ++ */ ++static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent, ++ const char *new_name_str) ++{ ++ struct cgroup *cgrp = kn->priv; ++ int ret; ++ ++ if (kernfs_type(kn) != KERNFS_DIR) ++ return -ENOTDIR; ++ if (kn->parent != new_parent) ++ return -EIO; ++ ++ /* ++ * This isn't a proper migration and its usefulness is very ++ * limited. Disallow on the default hierarchy. ++ */ ++ if (cgroup_on_dfl(cgrp)) ++ return -EPERM; ++ ++ /* ++ * We're gonna grab cgroup_mutex which nests outside kernfs ++ * active_ref. kernfs_rename() doesn't require active_ref ++ * protection. Break them before grabbing cgroup_mutex. ++ */ ++ kernfs_break_active_protection(new_parent); ++ kernfs_break_active_protection(kn); ++ ++ mutex_lock(&cgroup_mutex); ++ ++ ret = kernfs_rename(kn, new_parent, new_name_str); ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ kernfs_unbreak_active_protection(kn); ++ kernfs_unbreak_active_protection(new_parent); ++ return ret; ++} ++ ++/* set uid and gid of cgroup dirs and files to that of the creator */ ++static int cgroup_kn_set_ugid(struct kernfs_node *kn) ++{ ++ struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID, ++ .ia_uid = current_fsuid(), ++ .ia_gid = current_fsgid(), }; ++ ++ if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) && ++ gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID)) ++ return 0; ++ ++ return kernfs_setattr(kn, &iattr); ++} ++ ++static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft) ++{ ++ char name[CGROUP_FILE_NAME_MAX]; ++ struct kernfs_node *kn; ++ struct lock_class_key *key = NULL; ++ int ret; ++ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ key = &cft->lockdep_key; ++#endif ++ kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), ++ cgroup_file_mode(cft), 0, cft->kf_ops, cft, ++ NULL, key); ++ if (IS_ERR(kn)) ++ return PTR_ERR(kn); ++ ++ ret = cgroup_kn_set_ugid(kn); ++ if (ret) { ++ kernfs_remove(kn); ++ return ret; ++ } ++ ++ if (cft->seq_show == cgroup_populated_show) ++ cgrp->populated_kn = kn; ++ return 0; ++} ++ ++/** ++ * cgroup_addrm_files - add or remove files to a cgroup directory ++ * @cgrp: the target cgroup ++ * @cfts: array of cftypes to be added ++ * @is_add: whether to add or remove ++ * ++ * Depending on @is_add, add or remove files defined by @cfts on @cgrp. ++ * For removals, this function never fails. If addition fails, this ++ * function doesn't remove files already added. The caller is responsible ++ * for cleaning up. ++ */ ++static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[], ++ bool is_add) ++{ ++ struct cftype *cft; ++ int ret; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ for (cft = cfts; cft->name[0] != '\0'; cft++) { ++ /* does cft->flags tell us to skip this file on @cgrp? */ ++ if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) ++ continue; ++ if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) ++ continue; ++ if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) ++ continue; ++ if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) ++ continue; ++ ++ if (is_add) { ++ ret = cgroup_add_file(cgrp, cft); ++ if (ret) { ++ pr_warn("%s: failed to add %s, err=%d\n", ++ __func__, cft->name, ret); ++ return ret; ++ } ++ } else { ++ cgroup_rm_file(cgrp, cft); ++ } ++ } ++ return 0; ++} ++ ++static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add) ++{ ++ LIST_HEAD(pending); ++ struct cgroup_subsys *ss = cfts[0].ss; ++ struct cgroup *root = &ss->root->cgrp; ++ struct cgroup_subsys_state *css; ++ int ret = 0; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* add/rm files for all cgroups created before */ ++ css_for_each_descendant_pre(css, cgroup_css(root, ss)) { ++ struct cgroup *cgrp = css->cgroup; ++ ++ if (cgroup_is_dead(cgrp)) ++ continue; ++ ++ ret = cgroup_addrm_files(cgrp, cfts, is_add); ++ if (ret) ++ break; ++ } ++ ++ if (is_add && !ret) ++ kernfs_activate(root->kn); ++ return ret; ++} ++ ++static void cgroup_exit_cftypes(struct cftype *cfts) ++{ ++ struct cftype *cft; ++ ++ for (cft = cfts; cft->name[0] != '\0'; cft++) { ++ /* free copy for custom atomic_write_len, see init_cftypes() */ ++ if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) ++ kfree(cft->kf_ops); ++ cft->kf_ops = NULL; ++ cft->ss = NULL; ++ ++ /* revert flags set by cgroup core while adding @cfts */ ++ cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL); ++ } ++} ++ ++static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) ++{ ++ struct cftype *cft; ++ ++ for (cft = cfts; cft->name[0] != '\0'; cft++) { ++ struct kernfs_ops *kf_ops; ++ ++ WARN_ON(cft->ss || cft->kf_ops); ++ ++ if (cft->seq_start) ++ kf_ops = &cgroup_kf_ops; ++ else ++ kf_ops = &cgroup_kf_single_ops; ++ ++ /* ++ * Ugh... if @cft wants a custom max_write_len, we need to ++ * make a copy of kf_ops to set its atomic_write_len. ++ */ ++ if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) { ++ kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL); ++ if (!kf_ops) { ++ cgroup_exit_cftypes(cfts); ++ return -ENOMEM; ++ } ++ kf_ops->atomic_write_len = cft->max_write_len; ++ } ++ ++ cft->kf_ops = kf_ops; ++ cft->ss = ss; ++ } ++ ++ return 0; ++} ++ ++static int cgroup_rm_cftypes_locked(struct cftype *cfts) ++{ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ if (!cfts || !cfts[0].ss) ++ return -ENOENT; ++ ++ list_del(&cfts->node); ++ cgroup_apply_cftypes(cfts, false); ++ cgroup_exit_cftypes(cfts); ++ return 0; ++} ++ ++/** ++ * cgroup_rm_cftypes - remove an array of cftypes from a subsystem ++ * @cfts: zero-length name terminated array of cftypes ++ * ++ * Unregister @cfts. Files described by @cfts are removed from all ++ * existing cgroups and all future cgroups won't have them either. This ++ * function can be called anytime whether @cfts' subsys is attached or not. ++ * ++ * Returns 0 on successful unregistration, -ENOENT if @cfts is not ++ * registered. ++ */ ++int cgroup_rm_cftypes(struct cftype *cfts) ++{ ++ int ret; ++ ++ mutex_lock(&cgroup_mutex); ++ ret = cgroup_rm_cftypes_locked(cfts); ++ mutex_unlock(&cgroup_mutex); ++ return ret; ++} ++ ++/** ++ * cgroup_add_cftypes - add an array of cftypes to a subsystem ++ * @ss: target cgroup subsystem ++ * @cfts: zero-length name terminated array of cftypes ++ * ++ * Register @cfts to @ss. Files described by @cfts are created for all ++ * existing cgroups to which @ss is attached and all future cgroups will ++ * have them too. This function can be called anytime whether @ss is ++ * attached or not. ++ * ++ * Returns 0 on successful registration, -errno on failure. Note that this ++ * function currently returns 0 as long as @cfts registration is successful ++ * even if some file creation attempts on existing cgroups fail. ++ */ ++static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) ++{ ++ int ret; ++ ++ if (ss->disabled) ++ return 0; ++ ++ if (!cfts || cfts[0].name[0] == '\0') ++ return 0; ++ ++ ret = cgroup_init_cftypes(ss, cfts); ++ if (ret) ++ return ret; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ list_add_tail(&cfts->node, &ss->cfts); ++ ret = cgroup_apply_cftypes(cfts, true); ++ if (ret) ++ cgroup_rm_cftypes_locked(cfts); ++ ++ mutex_unlock(&cgroup_mutex); ++ return ret; ++} ++ ++/** ++ * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy ++ * @ss: target cgroup subsystem ++ * @cfts: zero-length name terminated array of cftypes ++ * ++ * Similar to cgroup_add_cftypes() but the added files are only used for ++ * the default hierarchy. ++ */ ++int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) ++{ ++ struct cftype *cft; ++ ++ for (cft = cfts; cft && cft->name[0] != '\0'; cft++) ++ cft->flags |= __CFTYPE_ONLY_ON_DFL; ++ return cgroup_add_cftypes(ss, cfts); ++} ++ ++/** ++ * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies ++ * @ss: target cgroup subsystem ++ * @cfts: zero-length name terminated array of cftypes ++ * ++ * Similar to cgroup_add_cftypes() but the added files are only used for ++ * the legacy hierarchies. ++ */ ++int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) ++{ ++ struct cftype *cft; ++ ++ /* ++ * If legacy_flies_on_dfl, we want to show the legacy files on the ++ * dfl hierarchy but iff the target subsystem hasn't been updated ++ * for the dfl hierarchy yet. ++ */ ++ if (!cgroup_legacy_files_on_dfl || ++ ss->dfl_cftypes != ss->legacy_cftypes) { ++ for (cft = cfts; cft && cft->name[0] != '\0'; cft++) ++ cft->flags |= __CFTYPE_NOT_ON_DFL; ++ } ++ ++ return cgroup_add_cftypes(ss, cfts); ++} ++ ++/** ++ * cgroup_task_count - count the number of tasks in a cgroup. ++ * @cgrp: the cgroup in question ++ * ++ * Return the number of tasks in the cgroup. ++ */ ++static int cgroup_task_count(const struct cgroup *cgrp) ++{ ++ int count = 0; ++ struct cgrp_cset_link *link; ++ ++ down_read(&css_set_rwsem); ++ list_for_each_entry(link, &cgrp->cset_links, cset_link) ++ count += atomic_read(&link->cset->refcount); ++ up_read(&css_set_rwsem); ++ return count; ++} ++ ++/** ++ * css_next_child - find the next child of a given css ++ * @pos: the current position (%NULL to initiate traversal) ++ * @parent: css whose children to walk ++ * ++ * This function returns the next child of @parent and should be called ++ * under either cgroup_mutex or RCU read lock. The only requirement is ++ * that @parent and @pos are accessible. The next sibling is guaranteed to ++ * be returned regardless of their states. ++ * ++ * If a subsystem synchronizes ->css_online() and the start of iteration, a ++ * css which finished ->css_online() is guaranteed to be visible in the ++ * future iterations and will stay visible until the last reference is put. ++ * A css which hasn't finished ->css_online() or already finished ++ * ->css_offline() may show up during traversal. It's each subsystem's ++ * responsibility to synchronize against on/offlining. ++ */ ++struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos, ++ struct cgroup_subsys_state *parent) ++{ ++ struct cgroup_subsys_state *next; ++ ++ cgroup_assert_mutex_or_rcu_locked(); ++ ++ /* ++ * @pos could already have been unlinked from the sibling list. ++ * Once a cgroup is removed, its ->sibling.next is no longer ++ * updated when its next sibling changes. CSS_RELEASED is set when ++ * @pos is taken off list, at which time its next pointer is valid, ++ * and, as releases are serialized, the one pointed to by the next ++ * pointer is guaranteed to not have started release yet. This ++ * implies that if we observe !CSS_RELEASED on @pos in this RCU ++ * critical section, the one pointed to by its next pointer is ++ * guaranteed to not have finished its RCU grace period even if we ++ * have dropped rcu_read_lock() inbetween iterations. ++ * ++ * If @pos has CSS_RELEASED set, its next pointer can't be ++ * dereferenced; however, as each css is given a monotonically ++ * increasing unique serial number and always appended to the ++ * sibling list, the next one can be found by walking the parent's ++ * children until the first css with higher serial number than ++ * @pos's. While this path can be slower, it happens iff iteration ++ * races against release and the race window is very small. ++ */ ++ if (!pos) { ++ next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling); ++ } else if (likely(!(pos->flags & CSS_RELEASED))) { ++ next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling); ++ } else { ++ list_for_each_entry_rcu(next, &parent->children, sibling) ++ if (next->serial_nr > pos->serial_nr) ++ break; ++ } ++ ++ /* ++ * @next, if not pointing to the head, can be dereferenced and is ++ * the next sibling. ++ */ ++ if (&next->sibling != &parent->children) ++ return next; ++ return NULL; ++} ++ ++/** ++ * css_next_descendant_pre - find the next descendant for pre-order walk ++ * @pos: the current position (%NULL to initiate traversal) ++ * @root: css whose descendants to walk ++ * ++ * To be used by css_for_each_descendant_pre(). Find the next descendant ++ * to visit for pre-order traversal of @root's descendants. @root is ++ * included in the iteration and the first node to be visited. ++ * ++ * While this function requires cgroup_mutex or RCU read locking, it ++ * doesn't require the whole traversal to be contained in a single critical ++ * section. This function will return the correct next descendant as long ++ * as both @pos and @root are accessible and @pos is a descendant of @root. ++ * ++ * If a subsystem synchronizes ->css_online() and the start of iteration, a ++ * css which finished ->css_online() is guaranteed to be visible in the ++ * future iterations and will stay visible until the last reference is put. ++ * A css which hasn't finished ->css_online() or already finished ++ * ->css_offline() may show up during traversal. It's each subsystem's ++ * responsibility to synchronize against on/offlining. ++ */ ++struct cgroup_subsys_state * ++css_next_descendant_pre(struct cgroup_subsys_state *pos, ++ struct cgroup_subsys_state *root) ++{ ++ struct cgroup_subsys_state *next; ++ ++ cgroup_assert_mutex_or_rcu_locked(); ++ ++ /* if first iteration, visit @root */ ++ if (!pos) ++ return root; ++ ++ /* visit the first child if exists */ ++ next = css_next_child(NULL, pos); ++ if (next) ++ return next; ++ ++ /* no child, visit my or the closest ancestor's next sibling */ ++ while (pos != root) { ++ next = css_next_child(pos, pos->parent); ++ if (next) ++ return next; ++ pos = pos->parent; ++ } ++ ++ return NULL; ++} ++ ++/** ++ * css_rightmost_descendant - return the rightmost descendant of a css ++ * @pos: css of interest ++ * ++ * Return the rightmost descendant of @pos. If there's no descendant, @pos ++ * is returned. This can be used during pre-order traversal to skip ++ * subtree of @pos. ++ * ++ * While this function requires cgroup_mutex or RCU read locking, it ++ * doesn't require the whole traversal to be contained in a single critical ++ * section. This function will return the correct rightmost descendant as ++ * long as @pos is accessible. ++ */ ++struct cgroup_subsys_state * ++css_rightmost_descendant(struct cgroup_subsys_state *pos) ++{ ++ struct cgroup_subsys_state *last, *tmp; ++ ++ cgroup_assert_mutex_or_rcu_locked(); ++ ++ do { ++ last = pos; ++ /* ->prev isn't RCU safe, walk ->next till the end */ ++ pos = NULL; ++ css_for_each_child(tmp, last) ++ pos = tmp; ++ } while (pos); ++ ++ return last; ++} ++ ++static struct cgroup_subsys_state * ++css_leftmost_descendant(struct cgroup_subsys_state *pos) ++{ ++ struct cgroup_subsys_state *last; ++ ++ do { ++ last = pos; ++ pos = css_next_child(NULL, pos); ++ } while (pos); ++ ++ return last; ++} ++ ++/** ++ * css_next_descendant_post - find the next descendant for post-order walk ++ * @pos: the current position (%NULL to initiate traversal) ++ * @root: css whose descendants to walk ++ * ++ * To be used by css_for_each_descendant_post(). Find the next descendant ++ * to visit for post-order traversal of @root's descendants. @root is ++ * included in the iteration and the last node to be visited. ++ * ++ * While this function requires cgroup_mutex or RCU read locking, it ++ * doesn't require the whole traversal to be contained in a single critical ++ * section. This function will return the correct next descendant as long ++ * as both @pos and @cgroup are accessible and @pos is a descendant of ++ * @cgroup. ++ * ++ * If a subsystem synchronizes ->css_online() and the start of iteration, a ++ * css which finished ->css_online() is guaranteed to be visible in the ++ * future iterations and will stay visible until the last reference is put. ++ * A css which hasn't finished ->css_online() or already finished ++ * ->css_offline() may show up during traversal. It's each subsystem's ++ * responsibility to synchronize against on/offlining. ++ */ ++struct cgroup_subsys_state * ++css_next_descendant_post(struct cgroup_subsys_state *pos, ++ struct cgroup_subsys_state *root) ++{ ++ struct cgroup_subsys_state *next; ++ ++ cgroup_assert_mutex_or_rcu_locked(); ++ ++ /* if first iteration, visit leftmost descendant which may be @root */ ++ if (!pos) ++ return css_leftmost_descendant(root); ++ ++ /* if we visited @root, we're done */ ++ if (pos == root) ++ return NULL; ++ ++ /* if there's an unvisited sibling, visit its leftmost descendant */ ++ next = css_next_child(pos, pos->parent); ++ if (next) ++ return css_leftmost_descendant(next); ++ ++ /* no sibling left, visit parent */ ++ return pos->parent; ++} ++ ++/** ++ * css_has_online_children - does a css have online children ++ * @css: the target css ++ * ++ * Returns %true if @css has any online children; otherwise, %false. This ++ * function can be called from any context but the caller is responsible ++ * for synchronizing against on/offlining as necessary. ++ */ ++bool css_has_online_children(struct cgroup_subsys_state *css) ++{ ++ struct cgroup_subsys_state *child; ++ bool ret = false; ++ ++ rcu_read_lock(); ++ css_for_each_child(child, css) { ++ if (child->flags & CSS_ONLINE) { ++ ret = true; ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ return ret; ++} ++ ++/** ++ * css_advance_task_iter - advance a task itererator to the next css_set ++ * @it: the iterator to advance ++ * ++ * Advance @it to the next css_set to walk. ++ */ ++static void css_advance_task_iter(struct css_task_iter *it) ++{ ++ struct list_head *l = it->cset_pos; ++ struct cgrp_cset_link *link; ++ struct css_set *cset; ++ ++ /* Advance to the next non-empty css_set */ ++ do { ++ l = l->next; ++ if (l == it->cset_head) { ++ it->cset_pos = NULL; ++ return; ++ } ++ ++ if (it->ss) { ++ cset = container_of(l, struct css_set, ++ e_cset_node[it->ss->id]); ++ } else { ++ link = list_entry(l, struct cgrp_cset_link, cset_link); ++ cset = link->cset; ++ } ++ } while (list_empty(&cset->tasks) && list_empty(&cset->mg_tasks)); ++ ++ it->cset_pos = l; ++ ++ if (!list_empty(&cset->tasks)) ++ it->task_pos = cset->tasks.next; ++ else ++ it->task_pos = cset->mg_tasks.next; ++ ++ it->tasks_head = &cset->tasks; ++ it->mg_tasks_head = &cset->mg_tasks; ++} ++ ++/** ++ * css_task_iter_start - initiate task iteration ++ * @css: the css to walk tasks of ++ * @it: the task iterator to use ++ * ++ * Initiate iteration through the tasks of @css. The caller can call ++ * css_task_iter_next() to walk through the tasks until the function ++ * returns NULL. On completion of iteration, css_task_iter_end() must be ++ * called. ++ * ++ * Note that this function acquires a lock which is released when the ++ * iteration finishes. The caller can't sleep while iteration is in ++ * progress. ++ */ ++void css_task_iter_start(struct cgroup_subsys_state *css, ++ struct css_task_iter *it) ++ __acquires(css_set_rwsem) ++{ ++ /* no one should try to iterate before mounting cgroups */ ++ WARN_ON_ONCE(!use_task_css_set_links); ++ ++ down_read(&css_set_rwsem); ++ ++ it->ss = css->ss; ++ ++ if (it->ss) ++ it->cset_pos = &css->cgroup->e_csets[css->ss->id]; ++ else ++ it->cset_pos = &css->cgroup->cset_links; ++ ++ it->cset_head = it->cset_pos; ++ ++ css_advance_task_iter(it); ++} ++ ++/** ++ * css_task_iter_next - return the next task for the iterator ++ * @it: the task iterator being iterated ++ * ++ * The "next" function for task iteration. @it should have been ++ * initialized via css_task_iter_start(). Returns NULL when the iteration ++ * reaches the end. ++ */ ++struct task_struct *css_task_iter_next(struct css_task_iter *it) ++{ ++ struct task_struct *res; ++ struct list_head *l = it->task_pos; ++ ++ /* If the iterator cg is NULL, we have no tasks */ ++ if (!it->cset_pos) ++ return NULL; ++ res = list_entry(l, struct task_struct, cg_list); ++ ++ /* ++ * Advance iterator to find next entry. cset->tasks is consumed ++ * first and then ->mg_tasks. After ->mg_tasks, we move onto the ++ * next cset. ++ */ ++ l = l->next; ++ ++ if (l == it->tasks_head) ++ l = it->mg_tasks_head->next; ++ ++ if (l == it->mg_tasks_head) ++ css_advance_task_iter(it); ++ else ++ it->task_pos = l; ++ ++ return res; ++} ++ ++/** ++ * css_task_iter_end - finish task iteration ++ * @it: the task iterator to finish ++ * ++ * Finish task iteration started by css_task_iter_start(). ++ */ ++void css_task_iter_end(struct css_task_iter *it) ++ __releases(css_set_rwsem) ++{ ++ up_read(&css_set_rwsem); ++} ++ ++/** ++ * cgroup_trasnsfer_tasks - move tasks from one cgroup to another ++ * @to: cgroup to which the tasks will be moved ++ * @from: cgroup in which the tasks currently reside ++ * ++ * Locking rules between cgroup_post_fork() and the migration path ++ * guarantee that, if a task is forking while being migrated, the new child ++ * is guaranteed to be either visible in the source cgroup after the ++ * parent's migration is complete or put into the target cgroup. No task ++ * can slip out of migration through forking. ++ */ ++int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from) ++{ ++ LIST_HEAD(preloaded_csets); ++ struct cgrp_cset_link *link; ++ struct css_task_iter it; ++ struct task_struct *task; ++ int ret; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ /* all tasks in @from are being moved, all csets are source */ ++ down_read(&css_set_rwsem); ++ list_for_each_entry(link, &from->cset_links, cset_link) ++ cgroup_migrate_add_src(link->cset, to, &preloaded_csets); ++ up_read(&css_set_rwsem); ++ ++ ret = cgroup_migrate_prepare_dst(to, &preloaded_csets); ++ if (ret) ++ goto out_err; ++ ++ /* ++ * Migrate tasks one-by-one until @form is empty. This fails iff ++ * ->can_attach() fails. ++ */ ++ do { ++ css_task_iter_start(&from->self, &it); ++ task = css_task_iter_next(&it); ++ if (task) ++ get_task_struct(task); ++ css_task_iter_end(&it); ++ ++ if (task) { ++ ret = cgroup_migrate(to, task, false); ++ put_task_struct(task); ++ } ++ } while (task && !ret); ++out_err: ++ cgroup_migrate_finish(&preloaded_csets); ++ mutex_unlock(&cgroup_mutex); ++ return ret; ++} ++ ++/* ++ * Stuff for reading the 'tasks'/'procs' files. ++ * ++ * Reading this file can return large amounts of data if a cgroup has ++ * *lots* of attached tasks. So it may need several calls to read(), ++ * but we cannot guarantee that the information we produce is correct ++ * unless we produce it entirely atomically. ++ * ++ */ ++ ++/* which pidlist file are we talking about? */ ++enum cgroup_filetype { ++ CGROUP_FILE_PROCS, ++ CGROUP_FILE_TASKS, ++}; ++ ++/* ++ * A pidlist is a list of pids that virtually represents the contents of one ++ * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists, ++ * a pair (one each for procs, tasks) for each pid namespace that's relevant ++ * to the cgroup. ++ */ ++struct cgroup_pidlist { ++ /* ++ * used to find which pidlist is wanted. doesn't change as long as ++ * this particular list stays in the list. ++ */ ++ struct { enum cgroup_filetype type; struct pid_namespace *ns; } key; ++ /* array of xids */ ++ pid_t *list; ++ /* how many elements the above list has */ ++ int length; ++ /* each of these stored in a list by its cgroup */ ++ struct list_head links; ++ /* pointer to the cgroup we belong to, for list removal purposes */ ++ struct cgroup *owner; ++ /* for delayed destruction */ ++ struct delayed_work destroy_dwork; ++}; ++ ++/* ++ * The following two functions "fix" the issue where there are more pids ++ * than kmalloc will give memory for; in such cases, we use vmalloc/vfree. ++ * TODO: replace with a kernel-wide solution to this problem ++ */ ++#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2)) ++static void *pidlist_allocate(int count) ++{ ++ if (PIDLIST_TOO_LARGE(count)) ++ return vmalloc(count * sizeof(pid_t)); ++ else ++ return kmalloc(count * sizeof(pid_t), GFP_KERNEL); ++} ++ ++static void pidlist_free(void *p) ++{ ++ kvfree(p); ++} ++ ++/* ++ * Used to destroy all pidlists lingering waiting for destroy timer. None ++ * should be left afterwards. ++ */ ++static void cgroup_pidlist_destroy_all(struct cgroup *cgrp) ++{ ++ struct cgroup_pidlist *l, *tmp_l; ++ ++ mutex_lock(&cgrp->pidlist_mutex); ++ list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links) ++ mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0); ++ mutex_unlock(&cgrp->pidlist_mutex); ++ ++ flush_workqueue(cgroup_pidlist_destroy_wq); ++ BUG_ON(!list_empty(&cgrp->pidlists)); ++} ++ ++static void cgroup_pidlist_destroy_work_fn(struct work_struct *work) ++{ ++ struct delayed_work *dwork = to_delayed_work(work); ++ struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist, ++ destroy_dwork); ++ struct cgroup_pidlist *tofree = NULL; ++ ++ mutex_lock(&l->owner->pidlist_mutex); ++ ++ /* ++ * Destroy iff we didn't get queued again. The state won't change ++ * as destroy_dwork can only be queued while locked. ++ */ ++ if (!delayed_work_pending(dwork)) { ++ list_del(&l->links); ++ pidlist_free(l->list); ++ put_pid_ns(l->key.ns); ++ tofree = l; ++ } ++ ++ mutex_unlock(&l->owner->pidlist_mutex); ++ kfree(tofree); ++} ++ ++/* ++ * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries ++ * Returns the number of unique elements. ++ */ ++static int pidlist_uniq(pid_t *list, int length) ++{ ++ int src, dest = 1; ++ ++ /* ++ * we presume the 0th element is unique, so i starts at 1. trivial ++ * edge cases first; no work needs to be done for either ++ */ ++ if (length == 0 || length == 1) ++ return length; ++ /* src and dest walk down the list; dest counts unique elements */ ++ for (src = 1; src < length; src++) { ++ /* find next unique element */ ++ while (list[src] == list[src-1]) { ++ src++; ++ if (src == length) ++ goto after; ++ } ++ /* dest always points to where the next unique element goes */ ++ list[dest] = list[src]; ++ dest++; ++ } ++after: ++ return dest; ++} ++ ++/* ++ * The two pid files - task and cgroup.procs - guaranteed that the result ++ * is sorted, which forced this whole pidlist fiasco. As pid order is ++ * different per namespace, each namespace needs differently sorted list, ++ * making it impossible to use, for example, single rbtree of member tasks ++ * sorted by task pointer. As pidlists can be fairly large, allocating one ++ * per open file is dangerous, so cgroup had to implement shared pool of ++ * pidlists keyed by cgroup and namespace. ++ * ++ * All this extra complexity was caused by the original implementation ++ * committing to an entirely unnecessary property. In the long term, we ++ * want to do away with it. Explicitly scramble sort order if on the ++ * default hierarchy so that no such expectation exists in the new ++ * interface. ++ * ++ * Scrambling is done by swapping every two consecutive bits, which is ++ * non-identity one-to-one mapping which disturbs sort order sufficiently. ++ */ ++static pid_t pid_fry(pid_t pid) ++{ ++ unsigned a = pid & 0x55555555; ++ unsigned b = pid & 0xAAAAAAAA; ++ ++ return (a << 1) | (b >> 1); ++} ++ ++static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid) ++{ ++ if (cgroup_on_dfl(cgrp)) ++ return pid_fry(pid); ++ else ++ return pid; ++} ++ ++static int cmppid(const void *a, const void *b) ++{ ++ return *(pid_t *)a - *(pid_t *)b; ++} ++ ++static int fried_cmppid(const void *a, const void *b) ++{ ++ return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b); ++} ++ ++static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp, ++ enum cgroup_filetype type) ++{ ++ struct cgroup_pidlist *l; ++ /* don't need task_nsproxy() if we're looking at ourself */ ++ struct pid_namespace *ns = task_active_pid_ns(current); ++ ++ lockdep_assert_held(&cgrp->pidlist_mutex); ++ ++ list_for_each_entry(l, &cgrp->pidlists, links) ++ if (l->key.type == type && l->key.ns == ns) ++ return l; ++ return NULL; ++} ++ ++/* ++ * find the appropriate pidlist for our purpose (given procs vs tasks) ++ * returns with the lock on that pidlist already held, and takes care ++ * of the use count, or returns NULL with no locks held if we're out of ++ * memory. ++ */ ++static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp, ++ enum cgroup_filetype type) ++{ ++ struct cgroup_pidlist *l; ++ ++ lockdep_assert_held(&cgrp->pidlist_mutex); ++ ++ l = cgroup_pidlist_find(cgrp, type); ++ if (l) ++ return l; ++ ++ /* entry not found; create a new one */ ++ l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL); ++ if (!l) ++ return l; ++ ++ INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn); ++ l->key.type = type; ++ /* don't need task_nsproxy() if we're looking at ourself */ ++ l->key.ns = get_pid_ns(task_active_pid_ns(current)); ++ l->owner = cgrp; ++ list_add(&l->links, &cgrp->pidlists); ++ return l; ++} ++ ++/* ++ * Load a cgroup's pidarray with either procs' tgids or tasks' pids ++ */ ++static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type, ++ struct cgroup_pidlist **lp) ++{ ++ pid_t *array; ++ int length; ++ int pid, n = 0; /* used for populating the array */ ++ struct css_task_iter it; ++ struct task_struct *tsk; ++ struct cgroup_pidlist *l; ++ ++ lockdep_assert_held(&cgrp->pidlist_mutex); ++ ++ /* ++ * If cgroup gets more users after we read count, we won't have ++ * enough space - tough. This race is indistinguishable to the ++ * caller from the case that the additional cgroup users didn't ++ * show up until sometime later on. ++ */ ++ length = cgroup_task_count(cgrp); ++ array = pidlist_allocate(length); ++ if (!array) ++ return -ENOMEM; ++ /* now, populate the array */ ++ css_task_iter_start(&cgrp->self, &it); ++ while ((tsk = css_task_iter_next(&it))) { ++ if (unlikely(n == length)) ++ break; ++ /* get tgid or pid for procs or tasks file respectively */ ++ if (type == CGROUP_FILE_PROCS) ++ pid = task_tgid_vnr(tsk); ++ else ++ pid = task_pid_vnr(tsk); ++ if (pid > 0) /* make sure to only use valid results */ ++ array[n++] = pid; ++ } ++ css_task_iter_end(&it); ++ length = n; ++ /* now sort & (if procs) strip out duplicates */ ++ if (cgroup_on_dfl(cgrp)) ++ sort(array, length, sizeof(pid_t), fried_cmppid, NULL); ++ else ++ sort(array, length, sizeof(pid_t), cmppid, NULL); ++ if (type == CGROUP_FILE_PROCS) ++ length = pidlist_uniq(array, length); ++ ++ l = cgroup_pidlist_find_create(cgrp, type); ++ if (!l) { ++ pidlist_free(array); ++ return -ENOMEM; ++ } ++ ++ /* store array, freeing old if necessary */ ++ pidlist_free(l->list); ++ l->list = array; ++ l->length = length; ++ *lp = l; ++ return 0; ++} ++ ++/** ++ * cgroupstats_build - build and fill cgroupstats ++ * @stats: cgroupstats to fill information into ++ * @dentry: A dentry entry belonging to the cgroup for which stats have ++ * been requested. ++ * ++ * Build and fill cgroupstats so that taskstats can export it to user ++ * space. ++ */ ++int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry) ++{ ++ struct kernfs_node *kn = kernfs_node_from_dentry(dentry); ++ struct cgroup *cgrp; ++ struct css_task_iter it; ++ struct task_struct *tsk; ++ ++ /* it should be kernfs_node belonging to cgroupfs and is a directory */ ++ if (dentry->d_sb->s_type != &cgroup_fs_type || !kn || ++ kernfs_type(kn) != KERNFS_DIR) ++ return -EINVAL; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ /* ++ * We aren't being called from kernfs and there's no guarantee on ++ * @kn->priv's validity. For this and css_tryget_online_from_dir(), ++ * @kn->priv is RCU safe. Let's do the RCU dancing. ++ */ ++ rcu_read_lock(); ++ cgrp = rcu_dereference(kn->priv); ++ if (!cgrp || cgroup_is_dead(cgrp)) { ++ rcu_read_unlock(); ++ mutex_unlock(&cgroup_mutex); ++ return -ENOENT; ++ } ++ rcu_read_unlock(); ++ ++ css_task_iter_start(&cgrp->self, &it); ++ while ((tsk = css_task_iter_next(&it))) { ++ switch (tsk->state) { ++ case TASK_RUNNING: ++ stats->nr_running++; ++ break; ++ case TASK_INTERRUPTIBLE: ++ stats->nr_sleeping++; ++ break; ++ case TASK_UNINTERRUPTIBLE: ++ stats->nr_uninterruptible++; ++ break; ++ case TASK_STOPPED: ++ stats->nr_stopped++; ++ break; ++ default: ++ if (delayacct_is_task_waiting_on_io(tsk)) ++ stats->nr_io_wait++; ++ break; ++ } ++ } ++ css_task_iter_end(&it); ++ ++ mutex_unlock(&cgroup_mutex); ++ return 0; ++} ++ ++ ++/* ++ * seq_file methods for the tasks/procs files. The seq_file position is the ++ * next pid to display; the seq_file iterator is a pointer to the pid ++ * in the cgroup->l->list array. ++ */ ++ ++static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos) ++{ ++ /* ++ * Initially we receive a position value that corresponds to ++ * one more than the last pid shown (or 0 on the first call or ++ * after a seek to the start). Use a binary-search to find the ++ * next pid to display, if any ++ */ ++ struct kernfs_open_file *of = s->private; ++ struct cgroup *cgrp = seq_css(s)->cgroup; ++ struct cgroup_pidlist *l; ++ enum cgroup_filetype type = seq_cft(s)->private; ++ int index = 0, pid = *pos; ++ int *iter, ret; ++ ++ mutex_lock(&cgrp->pidlist_mutex); ++ ++ /* ++ * !NULL @of->priv indicates that this isn't the first start() ++ * after open. If the matching pidlist is around, we can use that. ++ * Look for it. Note that @of->priv can't be used directly. It ++ * could already have been destroyed. ++ */ ++ if (of->priv) ++ of->priv = cgroup_pidlist_find(cgrp, type); ++ ++ /* ++ * Either this is the first start() after open or the matching ++ * pidlist has been destroyed inbetween. Create a new one. ++ */ ++ if (!of->priv) { ++ ret = pidlist_array_load(cgrp, type, ++ (struct cgroup_pidlist **)&of->priv); ++ if (ret) ++ return ERR_PTR(ret); ++ } ++ l = of->priv; ++ ++ if (pid) { ++ int end = l->length; ++ ++ while (index < end) { ++ int mid = (index + end) / 2; ++ if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) { ++ index = mid; ++ break; ++ } else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid) ++ index = mid + 1; ++ else ++ end = mid; ++ } ++ } ++ /* If we're off the end of the array, we're done */ ++ if (index >= l->length) ++ return NULL; ++ /* Update the abstract position to be the actual pid that we found */ ++ iter = l->list + index; ++ *pos = cgroup_pid_fry(cgrp, *iter); ++ return iter; ++} ++ ++static void cgroup_pidlist_stop(struct seq_file *s, void *v) ++{ ++ struct kernfs_open_file *of = s->private; ++ struct cgroup_pidlist *l = of->priv; ++ ++ if (l) ++ mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, ++ CGROUP_PIDLIST_DESTROY_DELAY); ++ mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex); ++} ++ ++static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos) ++{ ++ struct kernfs_open_file *of = s->private; ++ struct cgroup_pidlist *l = of->priv; ++ pid_t *p = v; ++ pid_t *end = l->list + l->length; ++ /* ++ * Advance to the next pid in the array. If this goes off the ++ * end, we're done ++ */ ++ p++; ++ if (p >= end) { ++ return NULL; ++ } else { ++ *pos = cgroup_pid_fry(seq_css(s)->cgroup, *p); ++ return p; ++ } ++} ++ ++static int cgroup_pidlist_show(struct seq_file *s, void *v) ++{ ++ seq_printf(s, "%d\n", *(int *)v); ++ ++ return 0; ++} ++ ++static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return notify_on_release(css->cgroup); ++} ++ ++static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css, ++ struct cftype *cft, u64 val) ++{ ++ if (val) ++ set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); ++ else ++ clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); ++ return 0; ++} ++ ++static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); ++} ++ ++static int cgroup_clone_children_write(struct cgroup_subsys_state *css, ++ struct cftype *cft, u64 val) ++{ ++ if (val) ++ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); ++ else ++ clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); ++ return 0; ++} ++ ++/* cgroup core interface files for the default hierarchy */ ++static struct cftype cgroup_dfl_base_files[] = { ++ { ++ .name = "cgroup.procs", ++ .seq_start = cgroup_pidlist_start, ++ .seq_next = cgroup_pidlist_next, ++ .seq_stop = cgroup_pidlist_stop, ++ .seq_show = cgroup_pidlist_show, ++ .private = CGROUP_FILE_PROCS, ++ .write = cgroup_procs_write, ++ .mode = S_IRUGO | S_IWUSR, ++ }, ++ { ++ .name = "cgroup.controllers", ++ .flags = CFTYPE_ONLY_ON_ROOT, ++ .seq_show = cgroup_root_controllers_show, ++ }, ++ { ++ .name = "cgroup.controllers", ++ .flags = CFTYPE_NOT_ON_ROOT, ++ .seq_show = cgroup_controllers_show, ++ }, ++ { ++ .name = "cgroup.subtree_control", ++ .seq_show = cgroup_subtree_control_show, ++ .write = cgroup_subtree_control_write, ++ }, ++ { ++ .name = "cgroup.populated", ++ .flags = CFTYPE_NOT_ON_ROOT, ++ .seq_show = cgroup_populated_show, ++ }, ++ { } /* terminate */ ++}; ++ ++/* cgroup core interface files for the legacy hierarchies */ ++static struct cftype cgroup_legacy_base_files[] = { ++ { ++ .name = "cgroup.procs", ++ .seq_start = cgroup_pidlist_start, ++ .seq_next = cgroup_pidlist_next, ++ .seq_stop = cgroup_pidlist_stop, ++ .seq_show = cgroup_pidlist_show, ++ .private = CGROUP_FILE_PROCS, ++ .write = cgroup_procs_write, ++ .mode = S_IRUGO | S_IWUSR, ++ }, ++ { ++ .name = "cgroup.clone_children", ++ .read_u64 = cgroup_clone_children_read, ++ .write_u64 = cgroup_clone_children_write, ++ }, ++ { ++ .name = "cgroup.sane_behavior", ++ .flags = CFTYPE_ONLY_ON_ROOT, ++ .seq_show = cgroup_sane_behavior_show, ++ }, ++ { ++ .name = "tasks", ++ .seq_start = cgroup_pidlist_start, ++ .seq_next = cgroup_pidlist_next, ++ .seq_stop = cgroup_pidlist_stop, ++ .seq_show = cgroup_pidlist_show, ++ .private = CGROUP_FILE_TASKS, ++ .write = cgroup_tasks_write, ++ .mode = S_IRUGO | S_IWUSR, ++ }, ++ { ++ .name = "notify_on_release", ++ .read_u64 = cgroup_read_notify_on_release, ++ .write_u64 = cgroup_write_notify_on_release, ++ }, ++ { ++ .name = "release_agent", ++ .flags = CFTYPE_ONLY_ON_ROOT, ++ .seq_show = cgroup_release_agent_show, ++ .write = cgroup_release_agent_write, ++ .max_write_len = PATH_MAX - 1, ++ }, ++ { } /* terminate */ ++}; ++ ++/** ++ * cgroup_populate_dir - create subsys files in a cgroup directory ++ * @cgrp: target cgroup ++ * @subsys_mask: mask of the subsystem ids whose files should be added ++ * ++ * On failure, no file is added. ++ */ ++static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask) ++{ ++ struct cgroup_subsys *ss; ++ int i, ret = 0; ++ ++ /* process cftsets of each subsystem */ ++ for_each_subsys(ss, i) { ++ struct cftype *cfts; ++ ++ if (!(subsys_mask & (1 << i))) ++ continue; ++ ++ list_for_each_entry(cfts, &ss->cfts, node) { ++ ret = cgroup_addrm_files(cgrp, cfts, true); ++ if (ret < 0) ++ goto err; ++ } ++ } ++ return 0; ++err: ++ cgroup_clear_dir(cgrp, subsys_mask); ++ return ret; ++} ++ ++/* ++ * css destruction is four-stage process. ++ * ++ * 1. Destruction starts. Killing of the percpu_ref is initiated. ++ * Implemented in kill_css(). ++ * ++ * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs ++ * and thus css_tryget_online() is guaranteed to fail, the css can be ++ * offlined by invoking offline_css(). After offlining, the base ref is ++ * put. Implemented in css_killed_work_fn(). ++ * ++ * 3. When the percpu_ref reaches zero, the only possible remaining ++ * accessors are inside RCU read sections. css_release() schedules the ++ * RCU callback. ++ * ++ * 4. After the grace period, the css can be freed. Implemented in ++ * css_free_work_fn(). ++ * ++ * It is actually hairier because both step 2 and 4 require process context ++ * and thus involve punting to css->destroy_work adding two additional ++ * steps to the already complex sequence. ++ */ ++static void css_free_work_fn(struct work_struct *work) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(work, struct cgroup_subsys_state, destroy_work); ++ struct cgroup_subsys *ss = css->ss; ++ struct cgroup *cgrp = css->cgroup; ++ ++ percpu_ref_exit(&css->refcnt); ++ ++ if (ss) { ++ /* css free path */ ++ int id = css->id; ++ ++ if (css->parent) ++ css_put(css->parent); ++ ++ ss->css_free(css); ++ cgroup_idr_remove(&ss->css_idr, id); ++ cgroup_put(cgrp); ++ } else { ++ /* cgroup free path */ ++ atomic_dec(&cgrp->root->nr_cgrps); ++ cgroup_pidlist_destroy_all(cgrp); ++ cancel_work_sync(&cgrp->release_agent_work); ++ ++ if (cgroup_parent(cgrp)) { ++ /* ++ * We get a ref to the parent, and put the ref when ++ * this cgroup is being freed, so it's guaranteed ++ * that the parent won't be destroyed before its ++ * children. ++ */ ++ cgroup_put(cgroup_parent(cgrp)); ++ kernfs_put(cgrp->kn); ++ kfree(cgrp); ++ } else { ++ /* ++ * This is root cgroup's refcnt reaching zero, ++ * which indicates that the root should be ++ * released. ++ */ ++ cgroup_destroy_root(cgrp->root); ++ } ++ } ++} ++ ++static void css_free_rcu_fn(struct rcu_head *rcu_head) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(rcu_head, struct cgroup_subsys_state, rcu_head); ++ ++ INIT_WORK(&css->destroy_work, css_free_work_fn); ++ queue_work(cgroup_destroy_wq, &css->destroy_work); ++} ++ ++static void css_release_work_fn(struct work_struct *work) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(work, struct cgroup_subsys_state, destroy_work); ++ struct cgroup_subsys *ss = css->ss; ++ struct cgroup *cgrp = css->cgroup; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ css->flags |= CSS_RELEASED; ++ list_del_rcu(&css->sibling); ++ ++ if (ss) { ++ /* css release path */ ++ cgroup_idr_replace(&ss->css_idr, NULL, css->id); ++ if (ss->css_released) ++ ss->css_released(css); ++ } else { ++ /* cgroup release path */ ++ cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id); ++ cgrp->id = -1; ++ ++ /* ++ * There are two control paths which try to determine ++ * cgroup from dentry without going through kernfs - ++ * cgroupstats_build() and css_tryget_online_from_dir(). ++ * Those are supported by RCU protecting clearing of ++ * cgrp->kn->priv backpointer. ++ */ ++ RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, NULL); ++ } ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ call_rcu(&css->rcu_head, css_free_rcu_fn); ++} ++ ++static void css_release(struct percpu_ref *ref) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(ref, struct cgroup_subsys_state, refcnt); ++ ++ INIT_WORK(&css->destroy_work, css_release_work_fn); ++ queue_work(cgroup_destroy_wq, &css->destroy_work); ++} ++ ++static void init_and_link_css(struct cgroup_subsys_state *css, ++ struct cgroup_subsys *ss, struct cgroup *cgrp) ++{ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ cgroup_get(cgrp); ++ ++ memset(css, 0, sizeof(*css)); ++ css->cgroup = cgrp; ++ css->ss = ss; ++ INIT_LIST_HEAD(&css->sibling); ++ INIT_LIST_HEAD(&css->children); ++ css->serial_nr = css_serial_nr_next++; ++ ++ if (cgroup_parent(cgrp)) { ++ css->parent = cgroup_css(cgroup_parent(cgrp), ss); ++ css_get(css->parent); ++ } ++ ++ BUG_ON(cgroup_css(cgrp, ss)); ++} ++ ++/* invoke ->css_online() on a new CSS and mark it online if successful */ ++static int online_css(struct cgroup_subsys_state *css) ++{ ++ struct cgroup_subsys *ss = css->ss; ++ int ret = 0; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ if (ss->css_online) ++ ret = ss->css_online(css); ++ if (!ret) { ++ css->flags |= CSS_ONLINE; ++ rcu_assign_pointer(css->cgroup->subsys[ss->id], css); ++ } ++ return ret; ++} ++ ++/* if the CSS is online, invoke ->css_offline() on it and mark it offline */ ++static void offline_css(struct cgroup_subsys_state *css) ++{ ++ struct cgroup_subsys *ss = css->ss; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ if (!(css->flags & CSS_ONLINE)) ++ return; ++ ++ if (ss->css_offline) ++ ss->css_offline(css); ++ ++ css->flags &= ~CSS_ONLINE; ++ RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); ++ ++ wake_up_all(&css->cgroup->offline_waitq); ++} ++ ++/** ++ * create_css - create a cgroup_subsys_state ++ * @cgrp: the cgroup new css will be associated with ++ * @ss: the subsys of new css ++ * @visible: whether to create control knobs for the new css or not ++ * ++ * Create a new css associated with @cgrp - @ss pair. On success, the new ++ * css is online and installed in @cgrp with all interface files created if ++ * @visible. Returns 0 on success, -errno on failure. ++ */ ++static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss, ++ bool visible) ++{ ++ struct cgroup *parent = cgroup_parent(cgrp); ++ struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss); ++ struct cgroup_subsys_state *css; ++ int err; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ css = ss->css_alloc(parent_css); ++ if (IS_ERR(css)) ++ return PTR_ERR(css); ++ ++ init_and_link_css(css, ss, cgrp); ++ ++ err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL); ++ if (err) ++ goto err_free_css; ++ ++ err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_NOWAIT); ++ if (err < 0) ++ goto err_free_percpu_ref; ++ css->id = err; ++ ++ if (visible) { ++ err = cgroup_populate_dir(cgrp, 1 << ss->id); ++ if (err) ++ goto err_free_id; ++ } ++ ++ /* @css is ready to be brought online now, make it visible */ ++ list_add_tail_rcu(&css->sibling, &parent_css->children); ++ cgroup_idr_replace(&ss->css_idr, css, css->id); ++ ++ err = online_css(css); ++ if (err) ++ goto err_list_del; ++ ++ if (ss->broken_hierarchy && !ss->warned_broken_hierarchy && ++ cgroup_parent(parent)) { ++ pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n", ++ current->comm, current->pid, ss->name); ++ if (!strcmp(ss->name, "memory")) ++ pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n"); ++ ss->warned_broken_hierarchy = true; ++ } ++ ++ return 0; ++ ++err_list_del: ++ list_del_rcu(&css->sibling); ++ cgroup_clear_dir(css->cgroup, 1 << css->ss->id); ++err_free_id: ++ cgroup_idr_remove(&ss->css_idr, css->id); ++err_free_percpu_ref: ++ percpu_ref_exit(&css->refcnt); ++err_free_css: ++ call_rcu(&css->rcu_head, css_free_rcu_fn); ++ return err; ++} ++ ++static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, ++ umode_t mode) ++{ ++ struct cgroup *parent, *cgrp; ++ struct cgroup_root *root; ++ struct cgroup_subsys *ss; ++ struct kernfs_node *kn; ++ struct cftype *base_files; ++ int ssid, ret; ++ ++ /* Do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable. ++ */ ++ if (strchr(name, '\n')) ++ return -EINVAL; ++ ++ parent = cgroup_kn_lock_live(parent_kn); ++ if (!parent) ++ return -ENODEV; ++ root = parent->root; ++ ++ /* allocate the cgroup and its ID, 0 is reserved for the root */ ++ cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL); ++ if (!cgrp) { ++ ret = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); ++ if (ret) ++ goto out_free_cgrp; ++ ++ /* ++ * Temporarily set the pointer to NULL, so idr_find() won't return ++ * a half-baked cgroup. ++ */ ++ cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_NOWAIT); ++ if (cgrp->id < 0) { ++ ret = -ENOMEM; ++ goto out_cancel_ref; ++ } ++ ++ init_cgroup_housekeeping(cgrp); ++ ++ cgrp->self.parent = &parent->self; ++ cgrp->root = root; ++ ++ if (notify_on_release(parent)) ++ set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); ++ ++ if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags)) ++ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); ++ ++ /* create the directory */ ++ kn = kernfs_create_dir(parent->kn, name, mode, cgrp); ++ if (IS_ERR(kn)) { ++ ret = PTR_ERR(kn); ++ goto out_free_id; ++ } ++ cgrp->kn = kn; ++ ++ /* ++ * This extra ref will be put in cgroup_free_fn() and guarantees ++ * that @cgrp->kn is always accessible. ++ */ ++ kernfs_get(kn); ++ ++ cgrp->self.serial_nr = css_serial_nr_next++; ++ ++ /* allocation complete, commit to creation */ ++ list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); ++ atomic_inc(&root->nr_cgrps); ++ cgroup_get(parent); ++ ++ /* ++ * @cgrp is now fully operational. If something fails after this ++ * point, it'll be released via the normal destruction path. ++ */ ++ cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id); ++ ++ ret = cgroup_kn_set_ugid(kn); ++ if (ret) ++ goto out_destroy; ++ ++ if (cgroup_on_dfl(cgrp)) ++ base_files = cgroup_dfl_base_files; ++ else ++ base_files = cgroup_legacy_base_files; ++ ++ ret = cgroup_addrm_files(cgrp, base_files, true); ++ if (ret) ++ goto out_destroy; ++ ++ /* let's create and online css's */ ++ for_each_subsys(ss, ssid) { ++ if (parent->child_subsys_mask & (1 << ssid)) { ++ ret = create_css(cgrp, ss, ++ parent->subtree_control & (1 << ssid)); ++ if (ret) ++ goto out_destroy; ++ } ++ } ++ ++ /* ++ * On the default hierarchy, a child doesn't automatically inherit ++ * subtree_control from the parent. Each is configured manually. ++ */ ++ if (!cgroup_on_dfl(cgrp)) { ++ cgrp->subtree_control = parent->subtree_control; ++ cgroup_refresh_child_subsys_mask(cgrp); ++ } ++ ++ kernfs_activate(kn); ++ ++ ret = 0; ++ goto out_unlock; ++ ++out_free_id: ++ cgroup_idr_remove(&root->cgroup_idr, cgrp->id); ++out_cancel_ref: ++ percpu_ref_exit(&cgrp->self.refcnt); ++out_free_cgrp: ++ kfree(cgrp); ++out_unlock: ++ cgroup_kn_unlock(parent_kn); ++ return ret; ++ ++out_destroy: ++ cgroup_destroy_locked(cgrp); ++ goto out_unlock; ++} ++ ++/* ++ * This is called when the refcnt of a css is confirmed to be killed. ++ * css_tryget_online() is now guaranteed to fail. Tell the subsystem to ++ * initate destruction and put the css ref from kill_css(). ++ */ ++static void css_killed_work_fn(struct work_struct *work) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(work, struct cgroup_subsys_state, destroy_work); ++ ++ mutex_lock(&cgroup_mutex); ++ offline_css(css); ++ mutex_unlock(&cgroup_mutex); ++ ++ css_put(css); ++} ++ ++/* css kill confirmation processing requires process context, bounce */ ++static void css_killed_ref_fn(struct percpu_ref *ref) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(ref, struct cgroup_subsys_state, refcnt); ++ ++ INIT_WORK(&css->destroy_work, css_killed_work_fn); ++ queue_work(cgroup_destroy_wq, &css->destroy_work); ++} ++ ++/** ++ * kill_css - destroy a css ++ * @css: css to destroy ++ * ++ * This function initiates destruction of @css by removing cgroup interface ++ * files and putting its base reference. ->css_offline() will be invoked ++ * asynchronously once css_tryget_online() is guaranteed to fail and when ++ * the reference count reaches zero, @css will be released. ++ */ ++static void kill_css(struct cgroup_subsys_state *css) ++{ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* ++ * This must happen before css is disassociated with its cgroup. ++ * See seq_css() for details. ++ */ ++ cgroup_clear_dir(css->cgroup, 1 << css->ss->id); ++ ++ /* ++ * Killing would put the base ref, but we need to keep it alive ++ * until after ->css_offline(). ++ */ ++ css_get(css); ++ ++ /* ++ * cgroup core guarantees that, by the time ->css_offline() is ++ * invoked, no new css reference will be given out via ++ * css_tryget_online(). We can't simply call percpu_ref_kill() and ++ * proceed to offlining css's because percpu_ref_kill() doesn't ++ * guarantee that the ref is seen as killed on all CPUs on return. ++ * ++ * Use percpu_ref_kill_and_confirm() to get notifications as each ++ * css is confirmed to be seen as killed on all CPUs. ++ */ ++ percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn); ++} ++ ++/** ++ * cgroup_destroy_locked - the first stage of cgroup destruction ++ * @cgrp: cgroup to be destroyed ++ * ++ * css's make use of percpu refcnts whose killing latency shouldn't be ++ * exposed to userland and are RCU protected. Also, cgroup core needs to ++ * guarantee that css_tryget_online() won't succeed by the time ++ * ->css_offline() is invoked. To satisfy all the requirements, ++ * destruction is implemented in the following two steps. ++ * ++ * s1. Verify @cgrp can be destroyed and mark it dying. Remove all ++ * userland visible parts and start killing the percpu refcnts of ++ * css's. Set up so that the next stage will be kicked off once all ++ * the percpu refcnts are confirmed to be killed. ++ * ++ * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the ++ * rest of destruction. Once all cgroup references are gone, the ++ * cgroup is RCU-freed. ++ * ++ * This function implements s1. After this step, @cgrp is gone as far as ++ * the userland is concerned and a new cgroup with the same name may be ++ * created. As cgroup doesn't care about the names internally, this ++ * doesn't cause any problem. ++ */ ++static int cgroup_destroy_locked(struct cgroup *cgrp) ++ __releases(&cgroup_mutex) __acquires(&cgroup_mutex) ++{ ++ struct cgroup_subsys_state *css; ++ bool empty; ++ int ssid; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* ++ * css_set_rwsem synchronizes access to ->cset_links and prevents ++ * @cgrp from being removed while put_css_set() is in progress. ++ */ ++ down_read(&css_set_rwsem); ++ empty = list_empty(&cgrp->cset_links); ++ up_read(&css_set_rwsem); ++ if (!empty) ++ return -EBUSY; ++ ++ /* ++ * Make sure there's no live children. We can't test emptiness of ++ * ->self.children as dead children linger on it while being ++ * drained; otherwise, "rmdir parent/child parent" may fail. ++ */ ++ if (css_has_online_children(&cgrp->self)) ++ return -EBUSY; ++ ++ /* ++ * Mark @cgrp dead. This prevents further task migration and child ++ * creation by disabling cgroup_lock_live_group(). ++ */ ++ cgrp->self.flags &= ~CSS_ONLINE; ++ ++ /* initiate massacre of all css's */ ++ for_each_css(css, ssid, cgrp) ++ kill_css(css); ++ ++ /* ++ * Remove @cgrp directory along with the base files. @cgrp has an ++ * extra ref on its kn. ++ */ ++ kernfs_remove(cgrp->kn); ++ ++ check_for_release(cgroup_parent(cgrp)); ++ ++ /* put the base reference */ ++ percpu_ref_kill(&cgrp->self.refcnt); ++ ++ return 0; ++}; ++ ++static int cgroup_rmdir(struct kernfs_node *kn) ++{ ++ struct cgroup *cgrp; ++ int ret = 0; ++ ++ cgrp = cgroup_kn_lock_live(kn); ++ if (!cgrp) ++ return 0; ++ ++ ret = cgroup_destroy_locked(cgrp); ++ ++ cgroup_kn_unlock(kn); ++ return ret; ++} ++ ++static struct kernfs_syscall_ops cgroup_kf_syscall_ops = { ++ .remount_fs = cgroup_remount, ++ .show_options = cgroup_show_options, ++ .mkdir = cgroup_mkdir, ++ .rmdir = cgroup_rmdir, ++ .rename = cgroup_rename, ++}; ++ ++static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early) ++{ ++ struct cgroup_subsys_state *css; ++ ++ printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name); ++ ++ mutex_lock(&cgroup_mutex); ++ ++ idr_init(&ss->css_idr); ++ INIT_LIST_HEAD(&ss->cfts); ++ ++ /* Create the root cgroup state for this subsystem */ ++ ss->root = &cgrp_dfl_root; ++ css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss)); ++ /* We don't handle early failures gracefully */ ++ BUG_ON(IS_ERR(css)); ++ init_and_link_css(css, ss, &cgrp_dfl_root.cgrp); ++ ++ /* ++ * Root csses are never destroyed and we can't initialize ++ * percpu_ref during early init. Disable refcnting. ++ */ ++ css->flags |= CSS_NO_REF; ++ ++ if (early) { ++ /* allocation can't be done safely during early init */ ++ css->id = 1; ++ } else { ++ css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL); ++ BUG_ON(css->id < 0); ++ } ++ ++ /* Update the init_css_set to contain a subsys ++ * pointer to this state - since the subsystem is ++ * newly registered, all tasks and hence the ++ * init_css_set is in the subsystem's root cgroup. */ ++ init_css_set.subsys[ss->id] = css; ++ ++ need_forkexit_callback |= ss->fork || ss->exit; ++ ++ /* At system boot, before all subsystems have been ++ * registered, no tasks have been forked, so we don't ++ * need to invoke fork callbacks here. */ ++ BUG_ON(!list_empty(&init_task.tasks)); ++ ++ BUG_ON(online_css(css)); ++ ++ mutex_unlock(&cgroup_mutex); ++} ++ ++/** ++ * cgroup_init_early - cgroup initialization at system boot ++ * ++ * Initialize cgroups at system boot, and initialize any ++ * subsystems that request early init. ++ */ ++int __init cgroup_init_early(void) ++{ ++ static struct cgroup_sb_opts __initdata opts; ++ struct cgroup_subsys *ss; ++ int i; ++ ++ init_cgroup_root(&cgrp_dfl_root, &opts); ++ cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF; ++ ++ RCU_INIT_POINTER(init_task.cgroups, &init_css_set); ++ ++ for_each_subsys(ss, i) { ++ WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id, ++ "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n", ++ i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free, ++ ss->id, ss->name); ++ WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN, ++ "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]); ++ ++ ss->id = i; ++ ss->name = cgroup_subsys_name[i]; ++ ++ if (ss->early_init) ++ cgroup_init_subsys(ss, true); ++ } ++ return 0; ++} ++ ++/** ++ * cgroup_init - cgroup initialization ++ * ++ * Register cgroup filesystem and /proc file, and initialize ++ * any subsystems that didn't request early init. ++ */ ++int __init cgroup_init(void) ++{ ++ struct cgroup_subsys *ss; ++ unsigned long key; ++ int ssid, err; ++ ++ BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files)); ++ BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files)); ++ ++ mutex_lock(&cgroup_mutex); ++ ++ /* Add init_css_set to the hash table */ ++ key = css_set_hash(init_css_set.subsys); ++ hash_add(css_set_table, &init_css_set.hlist, key); ++ ++ BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0)); ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ for_each_subsys(ss, ssid) { ++ if (ss->early_init) { ++ struct cgroup_subsys_state *css = ++ init_css_set.subsys[ss->id]; ++ ++ css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, ++ GFP_KERNEL); ++ BUG_ON(css->id < 0); ++ } else { ++ cgroup_init_subsys(ss, false); ++ } ++ ++ list_add_tail(&init_css_set.e_cset_node[ssid], ++ &cgrp_dfl_root.cgrp.e_csets[ssid]); ++ ++ /* ++ * Setting dfl_root subsys_mask needs to consider the ++ * disabled flag and cftype registration needs kmalloc, ++ * both of which aren't available during early_init. ++ */ ++ if (ss->disabled) ++ continue; ++ ++ cgrp_dfl_root.subsys_mask |= 1 << ss->id; ++ ++ if (cgroup_legacy_files_on_dfl && !ss->dfl_cftypes) ++ ss->dfl_cftypes = ss->legacy_cftypes; ++ ++ if (!ss->dfl_cftypes) ++ cgrp_dfl_root_inhibit_ss_mask |= 1 << ss->id; ++ ++ if (ss->dfl_cftypes == ss->legacy_cftypes) { ++ WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes)); ++ } else { ++ WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes)); ++ WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes)); ++ } ++ ++ if (ss->bind) ++ ss->bind(init_css_set.subsys[ssid]); ++ } ++ ++ err = sysfs_create_mount_point(fs_kobj, "cgroup"); ++ if (err) ++ return err; ++ ++ err = register_filesystem(&cgroup_fs_type); ++ if (err < 0) { ++ sysfs_remove_mount_point(fs_kobj, "cgroup"); ++ return err; ++ } ++ ++ proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations); ++ return 0; ++} ++ ++static int __init cgroup_wq_init(void) ++{ ++ /* ++ * There isn't much point in executing destruction path in ++ * parallel. Good chunk is serialized with cgroup_mutex anyway. ++ * Use 1 for @max_active. ++ * ++ * We would prefer to do this in cgroup_init() above, but that ++ * is called before init_workqueues(): so leave this until after. ++ */ ++ cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); ++ BUG_ON(!cgroup_destroy_wq); ++ ++ /* ++ * Used to destroy pidlists and separate to serve as flush domain. ++ * Cap @max_active to 1 too. ++ */ ++ cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy", ++ 0, 1); ++ BUG_ON(!cgroup_pidlist_destroy_wq); ++ ++ return 0; ++} ++core_initcall(cgroup_wq_init); ++ ++/* ++ * proc_cgroup_show() ++ * - Print task's cgroup paths into seq_file, one line for each hierarchy ++ * - Used for /proc/<pid>/cgroup. ++ */ ++int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns, ++ struct pid *pid, struct task_struct *tsk) ++{ ++ char *buf, *path; ++ int retval; ++ struct cgroup_root *root; ++ ++ retval = -ENOMEM; ++ buf = kmalloc(PATH_MAX, GFP_KERNEL); ++ if (!buf) ++ goto out; ++ ++ mutex_lock(&cgroup_mutex); ++ down_read(&css_set_rwsem); ++ ++ for_each_root(root) { ++ struct cgroup_subsys *ss; ++ struct cgroup *cgrp; ++ int ssid, count = 0; ++ ++ if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible) ++ continue; ++ ++ seq_printf(m, "%d:", root->hierarchy_id); ++ for_each_subsys(ss, ssid) ++ if (root->subsys_mask & (1 << ssid)) ++ seq_printf(m, "%s%s", count++ ? "," : "", ss->name); ++ if (strlen(root->name)) ++ seq_printf(m, "%sname=%s", count ? "," : "", ++ root->name); ++ seq_putc(m, ':'); ++ cgrp = task_cgroup_from_root(tsk, root); ++ path = cgroup_path(cgrp, buf, PATH_MAX); ++ if (!path) { ++ retval = -ENAMETOOLONG; ++ goto out_unlock; ++ } ++ seq_puts(m, path); ++ seq_putc(m, '\n'); ++ } ++ ++ retval = 0; ++out_unlock: ++ up_read(&css_set_rwsem); ++ mutex_unlock(&cgroup_mutex); ++ kfree(buf); ++out: ++ return retval; ++} ++ ++/* Display information about each subsystem and each hierarchy */ ++static int proc_cgroupstats_show(struct seq_file *m, void *v) ++{ ++ struct cgroup_subsys *ss; ++ int i; ++ ++ seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n"); ++ /* ++ * ideally we don't want subsystems moving around while we do this. ++ * cgroup_mutex is also necessary to guarantee an atomic snapshot of ++ * subsys/hierarchy state. ++ */ ++ mutex_lock(&cgroup_mutex); ++ ++ for_each_subsys(ss, i) ++ seq_printf(m, "%s\t%d\t%d\t%d\n", ++ ss->name, ss->root->hierarchy_id, ++ atomic_read(&ss->root->nr_cgrps), !ss->disabled); ++ ++ mutex_unlock(&cgroup_mutex); ++ return 0; ++} ++ ++static int cgroupstats_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, proc_cgroupstats_show, NULL); ++} ++ ++static const struct file_operations proc_cgroupstats_operations = { ++ .open = cgroupstats_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++/** ++ * cgroup_fork - initialize cgroup related fields during copy_process() ++ * @child: pointer to task_struct of forking parent process. ++ * ++ * A task is associated with the init_css_set until cgroup_post_fork() ++ * attaches it to the parent's css_set. Empty cg_list indicates that ++ * @child isn't holding reference to its css_set. ++ */ ++void cgroup_fork(struct task_struct *child) ++{ ++ RCU_INIT_POINTER(child->cgroups, &init_css_set); ++ INIT_LIST_HEAD(&child->cg_list); ++} ++ ++/** ++ * cgroup_post_fork - called on a new task after adding it to the task list ++ * @child: the task in question ++ * ++ * Adds the task to the list running through its css_set if necessary and ++ * call the subsystem fork() callbacks. Has to be after the task is ++ * visible on the task list in case we race with the first call to ++ * cgroup_task_iter_start() - to guarantee that the new task ends up on its ++ * list. ++ */ ++void cgroup_post_fork(struct task_struct *child) ++{ ++ struct cgroup_subsys *ss; ++ int i; ++ ++ /* ++ * This may race against cgroup_enable_task_cg_lists(). As that ++ * function sets use_task_css_set_links before grabbing ++ * tasklist_lock and we just went through tasklist_lock to add ++ * @child, it's guaranteed that either we see the set ++ * use_task_css_set_links or cgroup_enable_task_cg_lists() sees ++ * @child during its iteration. ++ * ++ * If we won the race, @child is associated with %current's ++ * css_set. Grabbing css_set_rwsem guarantees both that the ++ * association is stable, and, on completion of the parent's ++ * migration, @child is visible in the source of migration or ++ * already in the destination cgroup. This guarantee is necessary ++ * when implementing operations which need to migrate all tasks of ++ * a cgroup to another. ++ * ++ * Note that if we lose to cgroup_enable_task_cg_lists(), @child ++ * will remain in init_css_set. This is safe because all tasks are ++ * in the init_css_set before cg_links is enabled and there's no ++ * operation which transfers all tasks out of init_css_set. ++ */ ++ if (use_task_css_set_links) { ++ struct css_set *cset; ++ ++ down_write(&css_set_rwsem); ++ cset = task_css_set(current); ++ if (list_empty(&child->cg_list)) { ++ rcu_assign_pointer(child->cgroups, cset); ++ list_add(&child->cg_list, &cset->tasks); ++ get_css_set(cset); ++ } ++ up_write(&css_set_rwsem); ++ } ++ ++ /* ++ * Call ss->fork(). This must happen after @child is linked on ++ * css_set; otherwise, @child might change state between ->fork() ++ * and addition to css_set. ++ */ ++ if (need_forkexit_callback) { ++ for_each_subsys(ss, i) ++ if (ss->fork) ++ ss->fork(child); ++ } ++} ++ ++/** ++ * cgroup_exit - detach cgroup from exiting task ++ * @tsk: pointer to task_struct of exiting process ++ * ++ * Description: Detach cgroup from @tsk and release it. ++ * ++ * Note that cgroups marked notify_on_release force every task in ++ * them to take the global cgroup_mutex mutex when exiting. ++ * This could impact scaling on very large systems. Be reluctant to ++ * use notify_on_release cgroups where very high task exit scaling ++ * is required on large systems. ++ * ++ * We set the exiting tasks cgroup to the root cgroup (top_cgroup). We ++ * call cgroup_exit() while the task is still competent to handle ++ * notify_on_release(), then leave the task attached to the root cgroup in ++ * each hierarchy for the remainder of its exit. No need to bother with ++ * init_css_set refcnting. init_css_set never goes away and we can't race ++ * with migration path - PF_EXITING is visible to migration path. ++ */ ++void cgroup_exit(struct task_struct *tsk) ++{ ++ struct cgroup_subsys *ss; ++ struct css_set *cset; ++ bool put_cset = false; ++ int i; ++ ++ /* ++ * Unlink from @tsk from its css_set. As migration path can't race ++ * with us, we can check cg_list without grabbing css_set_rwsem. ++ */ ++ if (!list_empty(&tsk->cg_list)) { ++ down_write(&css_set_rwsem); ++ list_del_init(&tsk->cg_list); ++ up_write(&css_set_rwsem); ++ put_cset = true; ++ } ++ ++ /* Reassign the task to the init_css_set. */ ++ cset = task_css_set(tsk); ++ RCU_INIT_POINTER(tsk->cgroups, &init_css_set); ++ ++ if (need_forkexit_callback) { ++ /* see cgroup_post_fork() for details */ ++ for_each_subsys(ss, i) { ++ if (ss->exit) { ++ struct cgroup_subsys_state *old_css = cset->subsys[i]; ++ struct cgroup_subsys_state *css = task_css(tsk, i); ++ ++ ss->exit(css, old_css, tsk); ++ } ++ } ++ } ++ ++ if (put_cset) ++ put_css_set(cset); ++} ++ ++static void check_for_release(struct cgroup *cgrp) ++{ ++ if (notify_on_release(cgrp) && !cgroup_has_tasks(cgrp) && ++ !css_has_online_children(&cgrp->self) && !cgroup_is_dead(cgrp)) ++ schedule_work(&cgrp->release_agent_work); ++} ++ ++/* ++ * Notify userspace when a cgroup is released, by running the ++ * configured release agent with the name of the cgroup (path ++ * relative to the root of cgroup file system) as the argument. ++ * ++ * Most likely, this user command will try to rmdir this cgroup. ++ * ++ * This races with the possibility that some other task will be ++ * attached to this cgroup before it is removed, or that some other ++ * user task will 'mkdir' a child cgroup of this cgroup. That's ok. ++ * The presumed 'rmdir' will fail quietly if this cgroup is no longer ++ * unused, and this cgroup will be reprieved from its death sentence, ++ * to continue to serve a useful existence. Next time it's released, ++ * we will get notified again, if it still has 'notify_on_release' set. ++ * ++ * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which ++ * means only wait until the task is successfully execve()'d. The ++ * separate release agent task is forked by call_usermodehelper(), ++ * then control in this thread returns here, without waiting for the ++ * release agent task. We don't bother to wait because the caller of ++ * this routine has no use for the exit status of the release agent ++ * task, so no sense holding our caller up for that. ++ */ ++static void cgroup_release_agent(struct work_struct *work) ++{ ++ struct cgroup *cgrp = ++ container_of(work, struct cgroup, release_agent_work); ++ char *pathbuf = NULL, *agentbuf = NULL, *path; ++ char *argv[3], *envp[3]; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); ++ agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL); ++ if (!pathbuf || !agentbuf) ++ goto out; ++ ++ path = cgroup_path(cgrp, pathbuf, PATH_MAX); ++ if (!path) ++ goto out; ++ ++ argv[0] = agentbuf; ++ argv[1] = path; ++ argv[2] = NULL; ++ ++ /* minimal command environment */ ++ envp[0] = "HOME=/"; ++ envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin"; ++ envp[2] = NULL; ++ ++ mutex_unlock(&cgroup_mutex); ++ call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); ++ goto out_free; ++out: ++ mutex_unlock(&cgroup_mutex); ++out_free: ++ kfree(agentbuf); ++ kfree(pathbuf); ++} ++ ++static int __init cgroup_disable(char *str) ++{ ++ struct cgroup_subsys *ss; ++ char *token; ++ int i; ++ ++ while ((token = strsep(&str, ",")) != NULL) { ++ if (!*token) ++ continue; ++ ++ for_each_subsys(ss, i) { ++ if (!strcmp(token, ss->name)) { ++ ss->disabled = 1; ++ printk(KERN_INFO "Disabling %s control group" ++ " subsystem\n", ss->name); ++ break; ++ } ++ } ++ } ++ return 1; ++} ++__setup("cgroup_disable=", cgroup_disable); ++ ++static int __init cgroup_set_legacy_files_on_dfl(char *str) ++{ ++ printk("cgroup: using legacy files on the default hierarchy\n"); ++ cgroup_legacy_files_on_dfl = true; ++ return 0; ++} ++__setup("cgroup__DEVEL__legacy_files_on_dfl", cgroup_set_legacy_files_on_dfl); ++ ++/** ++ * css_tryget_online_from_dir - get corresponding css from a cgroup dentry ++ * @dentry: directory dentry of interest ++ * @ss: subsystem of interest ++ * ++ * If @dentry is a directory for a cgroup which has @ss enabled on it, try ++ * to get the corresponding css and return it. If such css doesn't exist ++ * or can't be pinned, an ERR_PTR value is returned. ++ */ ++struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry, ++ struct cgroup_subsys *ss) ++{ ++ struct kernfs_node *kn = kernfs_node_from_dentry(dentry); ++ struct cgroup_subsys_state *css = NULL; ++ struct cgroup *cgrp; ++ ++ /* is @dentry a cgroup dir? */ ++ if (dentry->d_sb->s_type != &cgroup_fs_type || !kn || ++ kernfs_type(kn) != KERNFS_DIR) ++ return ERR_PTR(-EBADF); ++ ++ rcu_read_lock(); ++ ++ /* ++ * This path doesn't originate from kernfs and @kn could already ++ * have been or be removed at any point. @kn->priv is RCU ++ * protected for this access. See css_release_work_fn() for details. ++ */ ++ cgrp = rcu_dereference(kn->priv); ++ if (cgrp) ++ css = cgroup_css(cgrp, ss); ++ ++ if (!css || !css_tryget_online(css)) ++ css = ERR_PTR(-ENOENT); ++ ++ rcu_read_unlock(); ++ return css; ++} ++ ++/** ++ * css_from_id - lookup css by id ++ * @id: the cgroup id ++ * @ss: cgroup subsys to be looked into ++ * ++ * Returns the css if there's valid one with @id, otherwise returns NULL. ++ * Should be called under rcu_read_lock(). ++ */ ++struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss) ++{ ++ WARN_ON_ONCE(!rcu_read_lock_held()); ++ return id > 0 ? idr_find(&ss->css_idr, id) : NULL; ++} ++ ++#ifdef CONFIG_CGROUP_DEBUG ++static struct cgroup_subsys_state * ++debug_css_alloc(struct cgroup_subsys_state *parent_css) ++{ ++ struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL); ++ ++ if (!css) ++ return ERR_PTR(-ENOMEM); ++ ++ return css; ++} ++ ++static void debug_css_free(struct cgroup_subsys_state *css) ++{ ++ kfree(css); ++} ++ ++static u64 debug_taskcount_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return cgroup_task_count(css->cgroup); ++} ++ ++static u64 current_css_set_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return (u64)(unsigned long)current->cgroups; ++} ++ ++static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ u64 count; ++ ++ rcu_read_lock(); ++ count = atomic_read(&task_css_set(current)->refcount); ++ rcu_read_unlock(); ++ return count; ++} ++ ++static int current_css_set_cg_links_read(struct seq_file *seq, void *v) ++{ ++ struct cgrp_cset_link *link; ++ struct css_set *cset; ++ char *name_buf; ++ ++ name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL); ++ if (!name_buf) ++ return -ENOMEM; ++ ++ down_read(&css_set_rwsem); ++ rcu_read_lock(); ++ cset = rcu_dereference(current->cgroups); ++ list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { ++ struct cgroup *c = link->cgrp; ++ ++ cgroup_name(c, name_buf, NAME_MAX + 1); ++ seq_printf(seq, "Root %d group %s\n", ++ c->root->hierarchy_id, name_buf); ++ } ++ rcu_read_unlock(); ++ up_read(&css_set_rwsem); ++ kfree(name_buf); ++ return 0; ++} ++ ++#define MAX_TASKS_SHOWN_PER_CSS 25 ++static int cgroup_css_links_read(struct seq_file *seq, void *v) ++{ ++ struct cgroup_subsys_state *css = seq_css(seq); ++ struct cgrp_cset_link *link; ++ ++ down_read(&css_set_rwsem); ++ list_for_each_entry(link, &css->cgroup->cset_links, cset_link) { ++ struct css_set *cset = link->cset; ++ struct task_struct *task; ++ int count = 0; ++ ++ seq_printf(seq, "css_set %p\n", cset); ++ ++ list_for_each_entry(task, &cset->tasks, cg_list) { ++ if (count++ > MAX_TASKS_SHOWN_PER_CSS) ++ goto overflow; ++ seq_printf(seq, " task %d\n", task_pid_vnr(task)); ++ } ++ ++ list_for_each_entry(task, &cset->mg_tasks, cg_list) { ++ if (count++ > MAX_TASKS_SHOWN_PER_CSS) ++ goto overflow; ++ seq_printf(seq, " task %d\n", task_pid_vnr(task)); ++ } ++ continue; ++ overflow: ++ seq_puts(seq, " ...\n"); ++ } ++ up_read(&css_set_rwsem); ++ return 0; ++} ++ ++static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft) ++{ ++ return (!cgroup_has_tasks(css->cgroup) && ++ !css_has_online_children(&css->cgroup->self)); ++} ++ ++static struct cftype debug_files[] = { ++ { ++ .name = "taskcount", ++ .read_u64 = debug_taskcount_read, ++ }, ++ ++ { ++ .name = "current_css_set", ++ .read_u64 = current_css_set_read, ++ }, ++ ++ { ++ .name = "current_css_set_refcount", ++ .read_u64 = current_css_set_refcount_read, ++ }, ++ ++ { ++ .name = "current_css_set_cg_links", ++ .seq_show = current_css_set_cg_links_read, ++ }, ++ ++ { ++ .name = "cgroup_css_links", ++ .seq_show = cgroup_css_links_read, ++ }, ++ ++ { ++ .name = "releasable", ++ .read_u64 = releasable_read, ++ }, ++ ++ { } /* terminate */ ++}; ++ ++struct cgroup_subsys debug_cgrp_subsys = { ++ .css_alloc = debug_css_alloc, ++ .css_free = debug_css_free, ++ .legacy_cftypes = debug_files, ++}; ++#endif /* CONFIG_CGROUP_DEBUG */ +diff -Nur linux-4.1.10.orig/kernel/cpu.c linux-4.1.10/kernel/cpu.c +--- linux-4.1.10.orig/kernel/cpu.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/cpu.c 2015-10-07 18:00:08.000000000 +0200 @@ -74,8 +74,8 @@ #endif } cpu_hotplug = { @@ -13601,9 +29214,9 @@ diff -Nur linux-4.1.6.orig/kernel/cpu.c linux-4.1.6/kernel/cpu.c return err; } -diff -Nur linux-4.1.6.orig/kernel/debug/kdb/kdb_io.c linux-4.1.6/kernel/debug/kdb/kdb_io.c ---- linux-4.1.6.orig/kernel/debug/kdb/kdb_io.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/debug/kdb/kdb_io.c 2015-09-08 23:49:08.089869160 +0200 +diff -Nur linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c linux-4.1.10/kernel/debug/kdb/kdb_io.c +--- linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/debug/kdb/kdb_io.c 2015-10-07 18:00:08.000000000 +0200 @@ -554,7 +554,6 @@ int linecount; int colcount; @@ -13641,10 +29254,10 @@ diff -Nur linux-4.1.6.orig/kernel/debug/kdb/kdb_io.c linux-4.1.6/kernel/debug/kd return r; } -diff -Nur linux-4.1.6.orig/kernel/events/core.c linux-4.1.6/kernel/events/core.c ---- linux-4.1.6.orig/kernel/events/core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/events/core.c 2015-09-08 23:49:08.089869160 +0200 -@@ -6890,6 +6890,7 @@ +diff -Nur linux-4.1.10.orig/kernel/events/core.c linux-4.1.10/kernel/events/core.c +--- linux-4.1.10.orig/kernel/events/core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/events/core.c 2015-10-07 18:00:08.000000000 +0200 +@@ -6933,6 +6933,7 @@ hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); hwc->hrtimer.function = perf_swevent_hrtimer; @@ -13652,9 +29265,9 @@ diff -Nur linux-4.1.6.orig/kernel/events/core.c linux-4.1.6/kernel/events/core.c /* * Since hrtimers have a fixed rate, we can do a static freq->period -diff -Nur linux-4.1.6.orig/kernel/exit.c linux-4.1.6/kernel/exit.c ---- linux-4.1.6.orig/kernel/exit.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/exit.c 2015-09-08 23:49:08.089869160 +0200 +diff -Nur linux-4.1.10.orig/kernel/exit.c linux-4.1.10/kernel/exit.c +--- linux-4.1.10.orig/kernel/exit.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/exit.c 2015-10-07 18:00:08.000000000 +0200 @@ -144,7 +144,7 @@ * Do this under ->siglock, we can race with another thread * doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals. @@ -13664,9 +29277,9 @@ diff -Nur linux-4.1.6.orig/kernel/exit.c linux-4.1.6/kernel/exit.c tsk->sighand = NULL; spin_unlock(&sighand->siglock); -diff -Nur linux-4.1.6.orig/kernel/fork.c linux-4.1.6/kernel/fork.c ---- linux-4.1.6.orig/kernel/fork.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/fork.c 2015-09-08 23:49:08.089869160 +0200 +diff -Nur linux-4.1.10.orig/kernel/fork.c linux-4.1.10/kernel/fork.c +--- linux-4.1.10.orig/kernel/fork.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/fork.c 2015-10-07 18:00:08.000000000 +0200 @@ -108,7 +108,7 @@ DEFINE_PER_CPU(unsigned long, process_counts) = 0; @@ -13764,9 +29377,9 @@ diff -Nur linux-4.1.6.orig/kernel/fork.c linux-4.1.6/kernel/fork.c #ifdef CONFIG_LOCKDEP p->lockdep_depth = 0; /* no locks held yet */ p->curr_chain_key = 0; -diff -Nur linux-4.1.6.orig/kernel/futex.c linux-4.1.6/kernel/futex.c ---- linux-4.1.6.orig/kernel/futex.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/futex.c 2015-09-08 23:49:08.089869160 +0200 +diff -Nur linux-4.1.10.orig/kernel/futex.c linux-4.1.10/kernel/futex.c +--- linux-4.1.10.orig/kernel/futex.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/futex.c 2015-10-07 18:00:08.000000000 +0200 @@ -738,7 +738,9 @@ * task still owns the PI-state: */ @@ -14095,9 +29708,9 @@ diff -Nur linux-4.1.6.orig/kernel/futex.c linux-4.1.6/kernel/futex.c /* * Fixup the pi_state owner and possibly acquire the lock if we * haven't already. -diff -Nur linux-4.1.6.orig/kernel/irq/handle.c linux-4.1.6/kernel/irq/handle.c ---- linux-4.1.6.orig/kernel/irq/handle.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq/handle.c 2015-09-08 23:49:08.093868717 +0200 +diff -Nur linux-4.1.10.orig/kernel/irq/handle.c linux-4.1.10/kernel/irq/handle.c +--- linux-4.1.10.orig/kernel/irq/handle.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/handle.c 2015-10-07 18:00:08.000000000 +0200 @@ -133,6 +133,8 @@ irqreturn_t handle_irq_event_percpu(struct irq_desc *desc, struct irqaction *action) @@ -14120,9 +29733,9 @@ diff -Nur linux-4.1.6.orig/kernel/irq/handle.c linux-4.1.6/kernel/irq/handle.c if (!noirqdebug) note_interrupt(irq, desc, retval); -diff -Nur linux-4.1.6.orig/kernel/irq/manage.c linux-4.1.6/kernel/irq/manage.c ---- linux-4.1.6.orig/kernel/irq/manage.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq/manage.c 2015-09-08 23:49:08.093868717 +0200 +diff -Nur linux-4.1.10.orig/kernel/irq/manage.c linux-4.1.10/kernel/irq/manage.c +--- linux-4.1.10.orig/kernel/irq/manage.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/manage.c 2015-10-07 18:00:08.000000000 +0200 @@ -22,6 +22,7 @@ #include "internals.h" @@ -14255,7 +29868,30 @@ diff -Nur linux-4.1.6.orig/kernel/irq/manage.c linux-4.1.6/kernel/irq/manage.c } raw_spin_lock_irqsave(&desc->lock, flags); -@@ -825,7 +900,15 @@ +@@ -697,6 +772,12 @@ + return IRQ_NONE; + } + ++static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id) ++{ ++ WARN(1, "Secondary action handler called for irq %d\n", irq); ++ return IRQ_NONE; ++} ++ + static int irq_wait_for_interrupt(struct irqaction *action) + { + set_current_state(TASK_INTERRUPTIBLE); +@@ -723,7 +804,8 @@ + static void irq_finalize_oneshot(struct irq_desc *desc, + struct irqaction *action) + { +- if (!(desc->istate & IRQS_ONESHOT)) ++ if (!(desc->istate & IRQS_ONESHOT) || ++ action->handler == irq_forced_secondary_handler) + return; + again: + chip_bus_lock(desc); +@@ -825,7 +907,15 @@ local_bh_disable(); ret = action->thread_fn(action->irq, action->dev_id); irq_finalize_oneshot(desc, action); @@ -14272,9 +29908,31 @@ diff -Nur linux-4.1.6.orig/kernel/irq/manage.c linux-4.1.6/kernel/irq/manage.c return ret; } -@@ -908,6 +991,12 @@ +@@ -877,6 +967,18 @@ + irq_finalize_oneshot(desc, action); + } + ++static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action) ++{ ++ struct irqaction *secondary = action->secondary; ++ ++ if (WARN_ON_ONCE(!secondary)) ++ return; ++ ++ raw_spin_lock_irq(&desc->lock); ++ __irq_wake_thread(desc, secondary); ++ raw_spin_unlock_irq(&desc->lock); ++} ++ + /* + * Interrupt handler thread + */ +@@ -907,7 +1009,15 @@ + action_ret = handler_fn(desc, action); if (action_ret == IRQ_HANDLED) atomic_inc(&desc->threads_handled); ++ if (action_ret == IRQ_WAKE_THREAD) ++ irq_wake_secondary(desc, action); +#ifdef CONFIG_PREEMPT_RT_FULL + migrate_disable(); @@ -14285,7 +29943,168 @@ diff -Nur linux-4.1.6.orig/kernel/irq/manage.c linux-4.1.6/kernel/irq/manage.c wake_threads_waitq(desc); } -@@ -1221,6 +1310,9 @@ +@@ -951,20 +1061,36 @@ + } + EXPORT_SYMBOL_GPL(irq_wake_thread); + +-static void irq_setup_forced_threading(struct irqaction *new) ++static int irq_setup_forced_threading(struct irqaction *new) + { + if (!force_irqthreads) +- return; ++ return 0; + if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT)) +- return; ++ return 0; + + new->flags |= IRQF_ONESHOT; + +- if (!new->thread_fn) { +- set_bit(IRQTF_FORCED_THREAD, &new->thread_flags); +- new->thread_fn = new->handler; +- new->handler = irq_default_primary_handler; +- } ++ /* ++ * Handle the case where we have a real primary handler and a ++ * thread handler. We force thread them as well by creating a ++ * secondary action. ++ */ ++ if (new->handler != irq_default_primary_handler && new->thread_fn) { ++ /* Allocate the secondary action */ ++ new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL); ++ if (!new->secondary) ++ return -ENOMEM; ++ new->secondary->handler = irq_forced_secondary_handler; ++ new->secondary->thread_fn = new->thread_fn; ++ new->secondary->dev_id = new->dev_id; ++ new->secondary->irq = new->irq; ++ new->secondary->name = new->name; ++ } ++ /* Deal with the primary handler */ ++ set_bit(IRQTF_FORCED_THREAD, &new->thread_flags); ++ new->thread_fn = new->handler; ++ new->handler = irq_default_primary_handler; ++ return 0; + } + + static int irq_request_resources(struct irq_desc *desc) +@@ -984,6 +1110,48 @@ + c->irq_release_resources(d); + } + ++static int ++setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary) ++{ ++ struct task_struct *t; ++ struct sched_param param = { ++ .sched_priority = MAX_USER_RT_PRIO/2, ++ }; ++ ++ if (!secondary) { ++ t = kthread_create(irq_thread, new, "irq/%d-%s", irq, ++ new->name); ++ } else { ++ t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq, ++ new->name); ++ param.sched_priority += 1; ++ } ++ ++ if (IS_ERR(t)) ++ return PTR_ERR(t); ++ ++ sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m); ++ ++ /* ++ * We keep the reference to the task struct even if ++ * the thread dies to avoid that the interrupt code ++ * references an already freed task_struct. ++ */ ++ get_task_struct(t); ++ new->thread = t; ++ /* ++ * Tell the thread to set its affinity. This is ++ * important for shared interrupt handlers as we do ++ * not invoke setup_affinity() for the secondary ++ * handlers as everything is already set up. Even for ++ * interrupts marked with IRQF_NO_BALANCE this is ++ * correct as we want the thread to move to the cpu(s) ++ * on which the requesting code placed the interrupt. ++ */ ++ set_bit(IRQTF_AFFINITY, &new->thread_flags); ++ return 0; ++} ++ + /* + * Internal function to register an irqaction - typically used to + * allocate special interrupts that are part of the architecture. +@@ -1004,6 +1172,8 @@ + if (!try_module_get(desc->owner)) + return -ENODEV; + ++ new->irq = irq; ++ + /* + * Check whether the interrupt nests into another interrupt + * thread. +@@ -1021,8 +1191,11 @@ + */ + new->handler = irq_nested_primary_handler; + } else { +- if (irq_settings_can_thread(desc)) +- irq_setup_forced_threading(new); ++ if (irq_settings_can_thread(desc)) { ++ ret = irq_setup_forced_threading(new); ++ if (ret) ++ goto out_mput; ++ } + } + + /* +@@ -1031,37 +1204,14 @@ + * thread. + */ + if (new->thread_fn && !nested) { +- struct task_struct *t; +- static const struct sched_param param = { +- .sched_priority = MAX_USER_RT_PRIO/2, +- }; +- +- t = kthread_create(irq_thread, new, "irq/%d-%s", irq, +- new->name); +- if (IS_ERR(t)) { +- ret = PTR_ERR(t); ++ ret = setup_irq_thread(new, irq, false); ++ if (ret) + goto out_mput; ++ if (new->secondary) { ++ ret = setup_irq_thread(new->secondary, irq, true); ++ if (ret) ++ goto out_thread; + } +- +- sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m); +- +- /* +- * We keep the reference to the task struct even if +- * the thread dies to avoid that the interrupt code +- * references an already freed task_struct. +- */ +- get_task_struct(t); +- new->thread = t; +- /* +- * Tell the thread to set its affinity. This is +- * important for shared interrupt handlers as we do +- * not invoke setup_affinity() for the secondary +- * handlers as everything is already set up. Even for +- * interrupts marked with IRQF_NO_BALANCE this is +- * correct as we want the thread to move to the cpu(s) +- * on which the requesting code placed the interrupt. +- */ +- set_bit(IRQTF_AFFINITY, &new->thread_flags); + } + + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) { +@@ -1221,6 +1371,9 @@ irqd_set(&desc->irq_data, IRQD_NO_BALANCING); } @@ -14295,9 +30114,67 @@ diff -Nur linux-4.1.6.orig/kernel/irq/manage.c linux-4.1.6/kernel/irq/manage.c /* Set default affinity mask once everything is setup */ setup_affinity(irq, desc, mask); -diff -Nur linux-4.1.6.orig/kernel/irq/settings.h linux-4.1.6/kernel/irq/settings.h ---- linux-4.1.6.orig/kernel/irq/settings.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq/settings.h 2015-09-08 23:49:08.093868717 +0200 +@@ -1234,7 +1387,6 @@ + irq, nmsk, omsk); + } + +- new->irq = irq; + *old_ptr = new; + + irq_pm_install_action(desc, new); +@@ -1260,6 +1412,8 @@ + */ + if (new->thread) + wake_up_process(new->thread); ++ if (new->secondary) ++ wake_up_process(new->secondary->thread); + + register_irq_proc(irq, desc); + new->dir = NULL; +@@ -1290,6 +1444,13 @@ + kthread_stop(t); + put_task_struct(t); + } ++ if (new->secondary && new->secondary->thread) { ++ struct task_struct *t = new->secondary->thread; ++ ++ new->secondary->thread = NULL; ++ kthread_stop(t); ++ put_task_struct(t); ++ } + out_mput: + module_put(desc->owner); + return ret; +@@ -1397,9 +1558,14 @@ + if (action->thread) { + kthread_stop(action->thread); + put_task_struct(action->thread); ++ if (action->secondary && action->secondary->thread) { ++ kthread_stop(action->secondary->thread); ++ put_task_struct(action->secondary->thread); ++ } + } + + module_put(desc->owner); ++ kfree(action->secondary); + return action; + } + +@@ -1543,8 +1709,10 @@ + retval = __setup_irq(irq, desc, action); + chip_bus_sync_unlock(desc); + +- if (retval) ++ if (retval) { ++ kfree(action->secondary); + kfree(action); ++ } + + #ifdef CONFIG_DEBUG_SHIRQ_FIXME + if (!retval && (irqflags & IRQF_SHARED)) { +diff -Nur linux-4.1.10.orig/kernel/irq/settings.h linux-4.1.10/kernel/irq/settings.h +--- linux-4.1.10.orig/kernel/irq/settings.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/settings.h 2015-10-07 18:00:08.000000000 +0200 @@ -15,6 +15,7 @@ _IRQ_NESTED_THREAD = IRQ_NESTED_THREAD, _IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID, @@ -14331,9 +30208,9 @@ diff -Nur linux-4.1.6.orig/kernel/irq/settings.h linux-4.1.6/kernel/irq/settings static inline bool irq_settings_is_per_cpu(struct irq_desc *desc) { return desc->status_use_accessors & _IRQ_PER_CPU; -diff -Nur linux-4.1.6.orig/kernel/irq/spurious.c linux-4.1.6/kernel/irq/spurious.c ---- linux-4.1.6.orig/kernel/irq/spurious.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq/spurious.c 2015-09-08 23:49:08.093868717 +0200 +diff -Nur linux-4.1.10.orig/kernel/irq/spurious.c linux-4.1.10/kernel/irq/spurious.c +--- linux-4.1.10.orig/kernel/irq/spurious.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/spurious.c 2015-10-07 18:00:08.000000000 +0200 @@ -444,6 +444,10 @@ static int __init irqfixup_setup(char *str) @@ -14356,9 +30233,9 @@ diff -Nur linux-4.1.6.orig/kernel/irq/spurious.c linux-4.1.6/kernel/irq/spurious irqfixup = 2; printk(KERN_WARNING "Misrouted IRQ fixup and polling support " "enabled\n"); -diff -Nur linux-4.1.6.orig/kernel/irq_work.c linux-4.1.6/kernel/irq_work.c ---- linux-4.1.6.orig/kernel/irq_work.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq_work.c 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/irq_work.c linux-4.1.10/kernel/irq_work.c +--- linux-4.1.10.orig/kernel/irq_work.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq_work.c 2015-10-07 18:00:08.000000000 +0200 @@ -17,6 +17,7 @@ #include <linux/cpu.h> #include <linux/notifier.h> @@ -14462,9 +30339,9 @@ diff -Nur linux-4.1.6.orig/kernel/irq_work.c linux-4.1.6/kernel/irq_work.c } EXPORT_SYMBOL_GPL(irq_work_run); -diff -Nur linux-4.1.6.orig/kernel/Kconfig.locks linux-4.1.6/kernel/Kconfig.locks ---- linux-4.1.6.orig/kernel/Kconfig.locks 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/Kconfig.locks 2015-09-08 23:49:08.085869602 +0200 +diff -Nur linux-4.1.10.orig/kernel/Kconfig.locks linux-4.1.10/kernel/Kconfig.locks +--- linux-4.1.10.orig/kernel/Kconfig.locks 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/Kconfig.locks 2015-10-07 18:00:08.000000000 +0200 @@ -225,11 +225,11 @@ config MUTEX_SPIN_ON_OWNER @@ -14479,9 +30356,9 @@ diff -Nur linux-4.1.6.orig/kernel/Kconfig.locks linux-4.1.6/kernel/Kconfig.locks config LOCK_SPIN_ON_OWNER def_bool y -diff -Nur linux-4.1.6.orig/kernel/Kconfig.preempt linux-4.1.6/kernel/Kconfig.preempt ---- linux-4.1.6.orig/kernel/Kconfig.preempt 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/Kconfig.preempt 2015-09-08 23:49:08.085869602 +0200 +diff -Nur linux-4.1.10.orig/kernel/Kconfig.preempt linux-4.1.10/kernel/Kconfig.preempt +--- linux-4.1.10.orig/kernel/Kconfig.preempt 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/Kconfig.preempt 2015-10-07 18:00:08.000000000 +0200 @@ -1,3 +1,16 @@ +config PREEMPT + bool @@ -14534,9 +30411,9 @@ diff -Nur linux-4.1.6.orig/kernel/Kconfig.preempt linux-4.1.6/kernel/Kconfig.pre endchoice config PREEMPT_COUNT -diff -Nur linux-4.1.6.orig/kernel/ksysfs.c linux-4.1.6/kernel/ksysfs.c ---- linux-4.1.6.orig/kernel/ksysfs.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/ksysfs.c 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/ksysfs.c linux-4.1.10/kernel/ksysfs.c +--- linux-4.1.10.orig/kernel/ksysfs.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/ksysfs.c 2015-10-07 18:00:08.000000000 +0200 @@ -136,6 +136,15 @@ #endif /* CONFIG_KEXEC */ @@ -14563,9 +30440,9 @@ diff -Nur linux-4.1.6.orig/kernel/ksysfs.c linux-4.1.6/kernel/ksysfs.c NULL }; -diff -Nur linux-4.1.6.orig/kernel/locking/lglock.c linux-4.1.6/kernel/locking/lglock.c ---- linux-4.1.6.orig/kernel/locking/lglock.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/lglock.c 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/locking/lglock.c linux-4.1.10/kernel/locking/lglock.c +--- linux-4.1.10.orig/kernel/locking/lglock.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/lglock.c 2015-10-07 18:00:08.000000000 +0200 @@ -4,6 +4,15 @@ #include <linux/cpu.h> #include <linux/string.h> @@ -14710,9 +30587,9 @@ diff -Nur linux-4.1.6.orig/kernel/locking/lglock.c linux-4.1.6/kernel/locking/lg + } +} +#endif -diff -Nur linux-4.1.6.orig/kernel/locking/lockdep.c linux-4.1.6/kernel/locking/lockdep.c ---- linux-4.1.6.orig/kernel/locking/lockdep.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/lockdep.c 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/locking/lockdep.c linux-4.1.10/kernel/locking/lockdep.c +--- linux-4.1.10.orig/kernel/locking/lockdep.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/lockdep.c 2015-10-07 18:00:08.000000000 +0200 @@ -3563,6 +3563,7 @@ } } @@ -14729,9 +30606,9 @@ diff -Nur linux-4.1.6.orig/kernel/locking/lockdep.c linux-4.1.6/kernel/locking/l if (!debug_locks) print_irqtrace_events(current); -diff -Nur linux-4.1.6.orig/kernel/locking/locktorture.c linux-4.1.6/kernel/locking/locktorture.c ---- linux-4.1.6.orig/kernel/locking/locktorture.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/locktorture.c 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/locking/locktorture.c linux-4.1.10/kernel/locking/locktorture.c +--- linux-4.1.10.orig/kernel/locking/locktorture.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/locktorture.c 2015-10-07 18:00:08.000000000 +0200 @@ -24,7 +24,6 @@ #include <linux/module.h> #include <linux/kthread.h> @@ -14740,9 +30617,9 @@ diff -Nur linux-4.1.6.orig/kernel/locking/locktorture.c linux-4.1.6/kernel/locki #include <linux/mutex.h> #include <linux/rwsem.h> #include <linux/smp.h> -diff -Nur linux-4.1.6.orig/kernel/locking/Makefile linux-4.1.6/kernel/locking/Makefile ---- linux-4.1.6.orig/kernel/locking/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/Makefile 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/locking/Makefile linux-4.1.10/kernel/locking/Makefile +--- linux-4.1.10.orig/kernel/locking/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/Makefile 2015-10-07 18:00:08.000000000 +0200 @@ -1,5 +1,5 @@ -obj-y += mutex.o semaphore.o rwsem.o @@ -14774,9 +30651,9 @@ diff -Nur linux-4.1.6.orig/kernel/locking/Makefile linux-4.1.6/kernel/locking/Ma +obj-$(CONFIG_PREEMPT_RT_FULL) += rt.o obj-$(CONFIG_QUEUE_RWLOCK) += qrwlock.o obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o -diff -Nur linux-4.1.6.orig/kernel/locking/rt.c linux-4.1.6/kernel/locking/rt.c ---- linux-4.1.6.orig/kernel/locking/rt.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/kernel/locking/rt.c 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/locking/rt.c linux-4.1.10/kernel/locking/rt.c +--- linux-4.1.10.orig/kernel/locking/rt.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/locking/rt.c 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,461 @@ +/* + * kernel/rt.c @@ -15239,9 +31116,9 @@ diff -Nur linux-4.1.6.orig/kernel/locking/rt.c linux-4.1.6/kernel/locking/rt.c + return 1; +} +EXPORT_SYMBOL(atomic_dec_and_mutex_lock); -diff -Nur linux-4.1.6.orig/kernel/locking/rtmutex.c linux-4.1.6/kernel/locking/rtmutex.c ---- linux-4.1.6.orig/kernel/locking/rtmutex.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/rtmutex.c 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex.c linux-4.1.10/kernel/locking/rtmutex.c +--- linux-4.1.10.orig/kernel/locking/rtmutex.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/rtmutex.c 2015-10-07 18:00:08.000000000 +0200 @@ -7,6 +7,11 @@ * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com> * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt @@ -16330,9 +32207,9 @@ diff -Nur linux-4.1.6.orig/kernel/locking/rtmutex.c linux-4.1.6/kernel/locking/r +} +EXPORT_SYMBOL(ww_mutex_unlock); +#endif -diff -Nur linux-4.1.6.orig/kernel/locking/rtmutex_common.h linux-4.1.6/kernel/locking/rtmutex_common.h ---- linux-4.1.6.orig/kernel/locking/rtmutex_common.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/rtmutex_common.h 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex_common.h linux-4.1.10/kernel/locking/rtmutex_common.h +--- linux-4.1.10.orig/kernel/locking/rtmutex_common.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/rtmutex_common.h 2015-10-07 18:00:08.000000000 +0200 @@ -49,6 +49,7 @@ struct rb_node pi_tree_entry; struct task_struct *task; @@ -16376,9 +32253,9 @@ diff -Nur linux-4.1.6.orig/kernel/locking/rtmutex_common.h linux-4.1.6/kernel/lo +} + #endif -diff -Nur linux-4.1.6.orig/kernel/locking/spinlock.c linux-4.1.6/kernel/locking/spinlock.c ---- linux-4.1.6.orig/kernel/locking/spinlock.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/spinlock.c 2015-09-08 23:49:08.101867828 +0200 +diff -Nur linux-4.1.10.orig/kernel/locking/spinlock.c linux-4.1.10/kernel/locking/spinlock.c +--- linux-4.1.10.orig/kernel/locking/spinlock.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/spinlock.c 2015-10-07 18:00:08.000000000 +0200 @@ -124,8 +124,11 @@ * __[spin|read|write]_lock_bh() */ @@ -16409,9 +32286,9 @@ diff -Nur linux-4.1.6.orig/kernel/locking/spinlock.c linux-4.1.6/kernel/locking/ #ifdef CONFIG_DEBUG_LOCK_ALLOC void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) -diff -Nur linux-4.1.6.orig/kernel/locking/spinlock_debug.c linux-4.1.6/kernel/locking/spinlock_debug.c ---- linux-4.1.6.orig/kernel/locking/spinlock_debug.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/spinlock_debug.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/locking/spinlock_debug.c linux-4.1.10/kernel/locking/spinlock_debug.c +--- linux-4.1.10.orig/kernel/locking/spinlock_debug.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/spinlock_debug.c 2015-10-07 18:00:08.000000000 +0200 @@ -31,6 +31,7 @@ EXPORT_SYMBOL(__raw_spin_lock_init); @@ -16442,9 +32319,9 @@ diff -Nur linux-4.1.6.orig/kernel/locking/spinlock_debug.c linux-4.1.6/kernel/lo } + +#endif -diff -Nur linux-4.1.6.orig/kernel/panic.c linux-4.1.6/kernel/panic.c ---- linux-4.1.6.orig/kernel/panic.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/panic.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/panic.c linux-4.1.10/kernel/panic.c +--- linux-4.1.10.orig/kernel/panic.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/panic.c 2015-10-07 18:00:08.000000000 +0200 @@ -387,9 +387,11 @@ static int init_oops_id(void) @@ -16457,9 +32334,9 @@ diff -Nur linux-4.1.6.orig/kernel/panic.c linux-4.1.6/kernel/panic.c oops_id++; return 0; -diff -Nur linux-4.1.6.orig/kernel/power/hibernate.c linux-4.1.6/kernel/power/hibernate.c ---- linux-4.1.6.orig/kernel/power/hibernate.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/power/hibernate.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/power/hibernate.c linux-4.1.10/kernel/power/hibernate.c +--- linux-4.1.10.orig/kernel/power/hibernate.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/power/hibernate.c 2015-10-07 18:00:08.000000000 +0200 @@ -285,6 +285,8 @@ local_irq_disable(); @@ -16509,9 +32386,9 @@ diff -Nur linux-4.1.6.orig/kernel/power/hibernate.c linux-4.1.6/kernel/power/hib local_irq_enable(); enable_nonboot_cpus(); -diff -Nur linux-4.1.6.orig/kernel/power/suspend.c linux-4.1.6/kernel/power/suspend.c ---- linux-4.1.6.orig/kernel/power/suspend.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/power/suspend.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/power/suspend.c linux-4.1.10/kernel/power/suspend.c +--- linux-4.1.10.orig/kernel/power/suspend.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/power/suspend.c 2015-10-07 18:00:08.000000000 +0200 @@ -356,6 +356,8 @@ arch_suspend_disable_irqs(); BUG_ON(!irqs_disabled()); @@ -16530,9 +32407,9 @@ diff -Nur linux-4.1.6.orig/kernel/power/suspend.c linux-4.1.6/kernel/power/suspe arch_suspend_enable_irqs(); BUG_ON(irqs_disabled()); -diff -Nur linux-4.1.6.orig/kernel/printk/printk.c linux-4.1.6/kernel/printk/printk.c ---- linux-4.1.6.orig/kernel/printk/printk.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/printk/printk.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/printk/printk.c linux-4.1.10/kernel/printk/printk.c +--- linux-4.1.10.orig/kernel/printk/printk.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/printk/printk.c 2015-10-07 18:00:08.000000000 +0200 @@ -1163,6 +1163,7 @@ { char *text; @@ -16785,9 +32662,9 @@ diff -Nur linux-4.1.6.orig/kernel/printk/printk.c linux-4.1.6/kernel/printk/prin } console_locked = 0; -diff -Nur linux-4.1.6.orig/kernel/ptrace.c linux-4.1.6/kernel/ptrace.c ---- linux-4.1.6.orig/kernel/ptrace.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/ptrace.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/ptrace.c linux-4.1.10/kernel/ptrace.c +--- linux-4.1.10.orig/kernel/ptrace.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/ptrace.c 2015-10-07 18:00:08.000000000 +0200 @@ -129,7 +129,12 @@ spin_lock_irq(&task->sighand->siglock); @@ -16802,9 +32679,9 @@ diff -Nur linux-4.1.6.orig/kernel/ptrace.c linux-4.1.6/kernel/ptrace.c ret = true; } spin_unlock_irq(&task->sighand->siglock); -diff -Nur linux-4.1.6.orig/kernel/rcu/tree.c linux-4.1.6/kernel/rcu/tree.c ---- linux-4.1.6.orig/kernel/rcu/tree.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/rcu/tree.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/rcu/tree.c linux-4.1.10/kernel/rcu/tree.c +--- linux-4.1.10.orig/kernel/rcu/tree.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/tree.c 2015-10-07 18:00:08.000000000 +0200 @@ -56,6 +56,11 @@ #include <linux/random.h> #include <linux/ftrace_event.h> @@ -17092,9 +32969,9 @@ diff -Nur linux-4.1.6.orig/kernel/rcu/tree.c linux-4.1.6/kernel/rcu/tree.c /* * We don't need protection against CPU-hotplug here because -diff -Nur linux-4.1.6.orig/kernel/rcu/tree.h linux-4.1.6/kernel/rcu/tree.h ---- linux-4.1.6.orig/kernel/rcu/tree.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/rcu/tree.h 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/rcu/tree.h linux-4.1.10/kernel/rcu/tree.h +--- linux-4.1.10.orig/kernel/rcu/tree.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/tree.h 2015-10-07 18:00:08.000000000 +0200 @@ -27,6 +27,7 @@ #include <linux/threads.h> #include <linux/cpumask.h> @@ -17155,9 +33032,9 @@ diff -Nur linux-4.1.6.orig/kernel/rcu/tree.h linux-4.1.6/kernel/rcu/tree.h static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, struct rcu_node *rnp); #endif /* #ifdef CONFIG_RCU_BOOST */ -diff -Nur linux-4.1.6.orig/kernel/rcu/tree_plugin.h linux-4.1.6/kernel/rcu/tree_plugin.h ---- linux-4.1.6.orig/kernel/rcu/tree_plugin.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/rcu/tree_plugin.h 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/rcu/tree_plugin.h linux-4.1.10/kernel/rcu/tree_plugin.h +--- linux-4.1.10.orig/kernel/rcu/tree_plugin.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/tree_plugin.h 2015-10-07 18:00:08.000000000 +0200 @@ -24,27 +24,20 @@ * Paul E. McKenney <paulmck@linux.vnet.ibm.com> */ @@ -17492,9 +33369,9 @@ diff -Nur linux-4.1.6.orig/kernel/rcu/tree_plugin.h linux-4.1.6/kernel/rcu/tree_ rdp->nocb_follower_tail = &rdp->nocb_follower_head; } -diff -Nur linux-4.1.6.orig/kernel/rcu/update.c linux-4.1.6/kernel/rcu/update.c ---- linux-4.1.6.orig/kernel/rcu/update.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/rcu/update.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/rcu/update.c linux-4.1.10/kernel/rcu/update.c +--- linux-4.1.10.orig/kernel/rcu/update.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/update.c 2015-10-07 18:00:08.000000000 +0200 @@ -227,6 +227,7 @@ } EXPORT_SYMBOL_GPL(rcu_read_lock_held); @@ -17511,9 +33388,9 @@ diff -Nur linux-4.1.6.orig/kernel/rcu/update.c linux-4.1.6/kernel/rcu/update.c #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -diff -Nur linux-4.1.6.orig/kernel/relay.c linux-4.1.6/kernel/relay.c ---- linux-4.1.6.orig/kernel/relay.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/relay.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/relay.c linux-4.1.10/kernel/relay.c +--- linux-4.1.10.orig/kernel/relay.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/relay.c 2015-10-07 18:00:08.000000000 +0200 @@ -339,6 +339,10 @@ { struct rchan_buf *buf = (struct rchan_buf *)data; @@ -17549,9 +33426,9 @@ diff -Nur linux-4.1.6.orig/kernel/relay.c linux-4.1.6/kernel/relay.c } old = buf->data; -diff -Nur linux-4.1.6.orig/kernel/sched/completion.c linux-4.1.6/kernel/sched/completion.c ---- linux-4.1.6.orig/kernel/sched/completion.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/completion.c 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/completion.c linux-4.1.10/kernel/sched/completion.c +--- linux-4.1.10.orig/kernel/sched/completion.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/completion.c 2015-10-07 18:00:08.000000000 +0200 @@ -30,10 +30,10 @@ { unsigned long flags; @@ -17642,9 +33519,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/completion.c linux-4.1.6/kernel/sched/co return true; } EXPORT_SYMBOL(completion_done); -diff -Nur linux-4.1.6.orig/kernel/sched/core.c linux-4.1.6/kernel/sched/core.c ---- linux-4.1.6.orig/kernel/sched/core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/core.c 2015-09-08 23:49:08.109866942 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/core.c linux-4.1.10/kernel/sched/core.c +--- linux-4.1.10.orig/kernel/sched/core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/core.c 2015-10-07 18:00:08.000000000 +0200 @@ -282,7 +282,11 @@ * Number of tasks to iterate in a single balance run. * Limited because this is done with IRQs disabled. @@ -18362,7 +34239,7 @@ diff -Nur linux-4.1.6.orig/kernel/sched/core.c linux-4.1.6/kernel/sched/core.c break; #endif } -@@ -7266,7 +7610,8 @@ +@@ -7274,7 +7618,8 @@ #ifdef CONFIG_DEBUG_ATOMIC_SLEEP static inline int preempt_count_equals(int preempt_offset) { @@ -18372,9 +34249,8402 @@ diff -Nur linux-4.1.6.orig/kernel/sched/core.c linux-4.1.6/kernel/sched/core.c return (nested == preempt_offset); } -diff -Nur linux-4.1.6.orig/kernel/sched/cputime.c linux-4.1.6/kernel/sched/cputime.c ---- linux-4.1.6.orig/kernel/sched/cputime.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/cputime.c 2015-09-08 23:49:08.109866942 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/core.c.orig linux-4.1.10/kernel/sched/core.c.orig +--- linux-4.1.10.orig/kernel/sched/core.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/sched/core.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,8389 @@ ++/* ++ * kernel/sched/core.c ++ * ++ * Kernel scheduler and related syscalls ++ * ++ * Copyright (C) 1991-2002 Linus Torvalds ++ * ++ * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and ++ * make semaphores SMP safe ++ * 1998-11-19 Implemented schedule_timeout() and related stuff ++ * by Andrea Arcangeli ++ * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: ++ * hybrid priority-list and round-robin design with ++ * an array-switch method of distributing timeslices ++ * and per-CPU runqueues. Cleanups and useful suggestions ++ * by Davide Libenzi, preemptible kernel bits by Robert Love. ++ * 2003-09-03 Interactivity tuning by Con Kolivas. ++ * 2004-04-02 Scheduler domains code by Nick Piggin ++ * 2007-04-15 Work begun on replacing all interactivity tuning with a ++ * fair scheduling design by Con Kolivas. ++ * 2007-05-05 Load balancing (smp-nice) and other improvements ++ * by Peter Williams ++ * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith ++ * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri ++ * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins, ++ * Thomas Gleixner, Mike Kravetz ++ */ ++ ++#include <linux/mm.h> ++#include <linux/module.h> ++#include <linux/nmi.h> ++#include <linux/init.h> ++#include <linux/uaccess.h> ++#include <linux/highmem.h> ++#include <asm/mmu_context.h> ++#include <linux/interrupt.h> ++#include <linux/capability.h> ++#include <linux/completion.h> ++#include <linux/kernel_stat.h> ++#include <linux/debug_locks.h> ++#include <linux/perf_event.h> ++#include <linux/security.h> ++#include <linux/notifier.h> ++#include <linux/profile.h> ++#include <linux/freezer.h> ++#include <linux/vmalloc.h> ++#include <linux/blkdev.h> ++#include <linux/delay.h> ++#include <linux/pid_namespace.h> ++#include <linux/smp.h> ++#include <linux/threads.h> ++#include <linux/timer.h> ++#include <linux/rcupdate.h> ++#include <linux/cpu.h> ++#include <linux/cpuset.h> ++#include <linux/percpu.h> ++#include <linux/proc_fs.h> ++#include <linux/seq_file.h> ++#include <linux/sysctl.h> ++#include <linux/syscalls.h> ++#include <linux/times.h> ++#include <linux/tsacct_kern.h> ++#include <linux/kprobes.h> ++#include <linux/delayacct.h> ++#include <linux/unistd.h> ++#include <linux/pagemap.h> ++#include <linux/hrtimer.h> ++#include <linux/tick.h> ++#include <linux/debugfs.h> ++#include <linux/ctype.h> ++#include <linux/ftrace.h> ++#include <linux/slab.h> ++#include <linux/init_task.h> ++#include <linux/binfmts.h> ++#include <linux/context_tracking.h> ++#include <linux/compiler.h> ++ ++#include <asm/switch_to.h> ++#include <asm/tlb.h> ++#include <asm/irq_regs.h> ++#include <asm/mutex.h> ++#ifdef CONFIG_PARAVIRT ++#include <asm/paravirt.h> ++#endif ++ ++#include "sched.h" ++#include "../workqueue_internal.h" ++#include "../smpboot.h" ++ ++#define CREATE_TRACE_POINTS ++#include <trace/events/sched.h> ++ ++void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period) ++{ ++ unsigned long delta; ++ ktime_t soft, hard, now; ++ ++ for (;;) { ++ if (hrtimer_active(period_timer)) ++ break; ++ ++ now = hrtimer_cb_get_time(period_timer); ++ hrtimer_forward(period_timer, now, period); ++ ++ soft = hrtimer_get_softexpires(period_timer); ++ hard = hrtimer_get_expires(period_timer); ++ delta = ktime_to_ns(ktime_sub(hard, soft)); ++ __hrtimer_start_range_ns(period_timer, soft, delta, ++ HRTIMER_MODE_ABS_PINNED, 0); ++ } ++} ++ ++DEFINE_MUTEX(sched_domains_mutex); ++DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); ++ ++static void update_rq_clock_task(struct rq *rq, s64 delta); ++ ++void update_rq_clock(struct rq *rq) ++{ ++ s64 delta; ++ ++ lockdep_assert_held(&rq->lock); ++ ++ if (rq->clock_skip_update & RQCF_ACT_SKIP) ++ return; ++ ++ delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; ++ if (delta < 0) ++ return; ++ rq->clock += delta; ++ update_rq_clock_task(rq, delta); ++} ++ ++/* ++ * Debugging: various feature bits ++ */ ++ ++#define SCHED_FEAT(name, enabled) \ ++ (1UL << __SCHED_FEAT_##name) * enabled | ++ ++const_debug unsigned int sysctl_sched_features = ++#include "features.h" ++ 0; ++ ++#undef SCHED_FEAT ++ ++#ifdef CONFIG_SCHED_DEBUG ++#define SCHED_FEAT(name, enabled) \ ++ #name , ++ ++static const char * const sched_feat_names[] = { ++#include "features.h" ++}; ++ ++#undef SCHED_FEAT ++ ++static int sched_feat_show(struct seq_file *m, void *v) ++{ ++ int i; ++ ++ for (i = 0; i < __SCHED_FEAT_NR; i++) { ++ if (!(sysctl_sched_features & (1UL << i))) ++ seq_puts(m, "NO_"); ++ seq_printf(m, "%s ", sched_feat_names[i]); ++ } ++ seq_puts(m, "\n"); ++ ++ return 0; ++} ++ ++#ifdef HAVE_JUMP_LABEL ++ ++#define jump_label_key__true STATIC_KEY_INIT_TRUE ++#define jump_label_key__false STATIC_KEY_INIT_FALSE ++ ++#define SCHED_FEAT(name, enabled) \ ++ jump_label_key__##enabled , ++ ++struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { ++#include "features.h" ++}; ++ ++#undef SCHED_FEAT ++ ++static void sched_feat_disable(int i) ++{ ++ if (static_key_enabled(&sched_feat_keys[i])) ++ static_key_slow_dec(&sched_feat_keys[i]); ++} ++ ++static void sched_feat_enable(int i) ++{ ++ if (!static_key_enabled(&sched_feat_keys[i])) ++ static_key_slow_inc(&sched_feat_keys[i]); ++} ++#else ++static void sched_feat_disable(int i) { }; ++static void sched_feat_enable(int i) { }; ++#endif /* HAVE_JUMP_LABEL */ ++ ++static int sched_feat_set(char *cmp) ++{ ++ int i; ++ int neg = 0; ++ ++ if (strncmp(cmp, "NO_", 3) == 0) { ++ neg = 1; ++ cmp += 3; ++ } ++ ++ for (i = 0; i < __SCHED_FEAT_NR; i++) { ++ if (strcmp(cmp, sched_feat_names[i]) == 0) { ++ if (neg) { ++ sysctl_sched_features &= ~(1UL << i); ++ sched_feat_disable(i); ++ } else { ++ sysctl_sched_features |= (1UL << i); ++ sched_feat_enable(i); ++ } ++ break; ++ } ++ } ++ ++ return i; ++} ++ ++static ssize_t ++sched_feat_write(struct file *filp, const char __user *ubuf, ++ size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ char *cmp; ++ int i; ++ struct inode *inode; ++ ++ if (cnt > 63) ++ cnt = 63; ++ ++ if (copy_from_user(&buf, ubuf, cnt)) ++ return -EFAULT; ++ ++ buf[cnt] = 0; ++ cmp = strstrip(buf); ++ ++ /* Ensure the static_key remains in a consistent state */ ++ inode = file_inode(filp); ++ mutex_lock(&inode->i_mutex); ++ i = sched_feat_set(cmp); ++ mutex_unlock(&inode->i_mutex); ++ if (i == __SCHED_FEAT_NR) ++ return -EINVAL; ++ ++ *ppos += cnt; ++ ++ return cnt; ++} ++ ++static int sched_feat_open(struct inode *inode, struct file *filp) ++{ ++ return single_open(filp, sched_feat_show, NULL); ++} ++ ++static const struct file_operations sched_feat_fops = { ++ .open = sched_feat_open, ++ .write = sched_feat_write, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static __init int sched_init_debug(void) ++{ ++ debugfs_create_file("sched_features", 0644, NULL, NULL, ++ &sched_feat_fops); ++ ++ return 0; ++} ++late_initcall(sched_init_debug); ++#endif /* CONFIG_SCHED_DEBUG */ ++ ++/* ++ * Number of tasks to iterate in a single balance run. ++ * Limited because this is done with IRQs disabled. ++ */ ++const_debug unsigned int sysctl_sched_nr_migrate = 32; ++ ++/* ++ * period over which we average the RT time consumption, measured ++ * in ms. ++ * ++ * default: 1s ++ */ ++const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC; ++ ++/* ++ * period over which we measure -rt task cpu usage in us. ++ * default: 1s ++ */ ++unsigned int sysctl_sched_rt_period = 1000000; ++ ++__read_mostly int scheduler_running; ++ ++/* ++ * part of the period that we allow rt tasks to run in us. ++ * default: 0.95s ++ */ ++int sysctl_sched_rt_runtime = 950000; ++ ++/* cpus with isolated domains */ ++cpumask_var_t cpu_isolated_map; ++ ++/* ++ * this_rq_lock - lock this runqueue and disable interrupts. ++ */ ++static struct rq *this_rq_lock(void) ++ __acquires(rq->lock) ++{ ++ struct rq *rq; ++ ++ local_irq_disable(); ++ rq = this_rq(); ++ raw_spin_lock(&rq->lock); ++ ++ return rq; ++} ++ ++#ifdef CONFIG_SCHED_HRTICK ++/* ++ * Use HR-timers to deliver accurate preemption points. ++ */ ++ ++static void hrtick_clear(struct rq *rq) ++{ ++ if (hrtimer_active(&rq->hrtick_timer)) ++ hrtimer_cancel(&rq->hrtick_timer); ++} ++ ++/* ++ * High-resolution timer tick. ++ * Runs from hardirq context with interrupts disabled. ++ */ ++static enum hrtimer_restart hrtick(struct hrtimer *timer) ++{ ++ struct rq *rq = container_of(timer, struct rq, hrtick_timer); ++ ++ WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); ++ ++ raw_spin_lock(&rq->lock); ++ update_rq_clock(rq); ++ rq->curr->sched_class->task_tick(rq, rq->curr, 1); ++ raw_spin_unlock(&rq->lock); ++ ++ return HRTIMER_NORESTART; ++} ++ ++#ifdef CONFIG_SMP ++ ++static int __hrtick_restart(struct rq *rq) ++{ ++ struct hrtimer *timer = &rq->hrtick_timer; ++ ktime_t time = hrtimer_get_softexpires(timer); ++ ++ return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0); ++} ++ ++/* ++ * called from hardirq (IPI) context ++ */ ++static void __hrtick_start(void *arg) ++{ ++ struct rq *rq = arg; ++ ++ raw_spin_lock(&rq->lock); ++ __hrtick_restart(rq); ++ rq->hrtick_csd_pending = 0; ++ raw_spin_unlock(&rq->lock); ++} ++ ++/* ++ * Called to set the hrtick timer state. ++ * ++ * called with rq->lock held and irqs disabled ++ */ ++void hrtick_start(struct rq *rq, u64 delay) ++{ ++ struct hrtimer *timer = &rq->hrtick_timer; ++ ktime_t time; ++ s64 delta; ++ ++ /* ++ * Don't schedule slices shorter than 10000ns, that just ++ * doesn't make sense and can cause timer DoS. ++ */ ++ delta = max_t(s64, delay, 10000LL); ++ time = ktime_add_ns(timer->base->get_time(), delta); ++ ++ hrtimer_set_expires(timer, time); ++ ++ if (rq == this_rq()) { ++ __hrtick_restart(rq); ++ } else if (!rq->hrtick_csd_pending) { ++ smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); ++ rq->hrtick_csd_pending = 1; ++ } ++} ++ ++static int ++hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu) ++{ ++ int cpu = (int)(long)hcpu; ++ ++ switch (action) { ++ case CPU_UP_CANCELED: ++ case CPU_UP_CANCELED_FROZEN: ++ case CPU_DOWN_PREPARE: ++ case CPU_DOWN_PREPARE_FROZEN: ++ case CPU_DEAD: ++ case CPU_DEAD_FROZEN: ++ hrtick_clear(cpu_rq(cpu)); ++ return NOTIFY_OK; ++ } ++ ++ return NOTIFY_DONE; ++} ++ ++static __init void init_hrtick(void) ++{ ++ hotcpu_notifier(hotplug_hrtick, 0); ++} ++#else ++/* ++ * Called to set the hrtick timer state. ++ * ++ * called with rq->lock held and irqs disabled ++ */ ++void hrtick_start(struct rq *rq, u64 delay) ++{ ++ /* ++ * Don't schedule slices shorter than 10000ns, that just ++ * doesn't make sense. Rely on vruntime for fairness. ++ */ ++ delay = max_t(u64, delay, 10000LL); ++ __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, ++ HRTIMER_MODE_REL_PINNED, 0); ++} ++ ++static inline void init_hrtick(void) ++{ ++} ++#endif /* CONFIG_SMP */ ++ ++static void init_rq_hrtick(struct rq *rq) ++{ ++#ifdef CONFIG_SMP ++ rq->hrtick_csd_pending = 0; ++ ++ rq->hrtick_csd.flags = 0; ++ rq->hrtick_csd.func = __hrtick_start; ++ rq->hrtick_csd.info = rq; ++#endif ++ ++ hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ rq->hrtick_timer.function = hrtick; ++} ++#else /* CONFIG_SCHED_HRTICK */ ++static inline void hrtick_clear(struct rq *rq) ++{ ++} ++ ++static inline void init_rq_hrtick(struct rq *rq) ++{ ++} ++ ++static inline void init_hrtick(void) ++{ ++} ++#endif /* CONFIG_SCHED_HRTICK */ ++ ++/* ++ * cmpxchg based fetch_or, macro so it works for different integer types ++ */ ++#define fetch_or(ptr, val) \ ++({ typeof(*(ptr)) __old, __val = *(ptr); \ ++ for (;;) { \ ++ __old = cmpxchg((ptr), __val, __val | (val)); \ ++ if (__old == __val) \ ++ break; \ ++ __val = __old; \ ++ } \ ++ __old; \ ++}) ++ ++#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) ++/* ++ * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, ++ * this avoids any races wrt polling state changes and thereby avoids ++ * spurious IPIs. ++ */ ++static bool set_nr_and_not_polling(struct task_struct *p) ++{ ++ struct thread_info *ti = task_thread_info(p); ++ return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); ++} ++ ++/* ++ * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. ++ * ++ * If this returns true, then the idle task promises to call ++ * sched_ttwu_pending() and reschedule soon. ++ */ ++static bool set_nr_if_polling(struct task_struct *p) ++{ ++ struct thread_info *ti = task_thread_info(p); ++ typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags); ++ ++ for (;;) { ++ if (!(val & _TIF_POLLING_NRFLAG)) ++ return false; ++ if (val & _TIF_NEED_RESCHED) ++ return true; ++ old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); ++ if (old == val) ++ break; ++ val = old; ++ } ++ return true; ++} ++ ++#else ++static bool set_nr_and_not_polling(struct task_struct *p) ++{ ++ set_tsk_need_resched(p); ++ return true; ++} ++ ++#ifdef CONFIG_SMP ++static bool set_nr_if_polling(struct task_struct *p) ++{ ++ return false; ++} ++#endif ++#endif ++ ++/* ++ * resched_curr - mark rq's current task 'to be rescheduled now'. ++ * ++ * On UP this means the setting of the need_resched flag, on SMP it ++ * might also involve a cross-CPU call to trigger the scheduler on ++ * the target CPU. ++ */ ++void resched_curr(struct rq *rq) ++{ ++ struct task_struct *curr = rq->curr; ++ int cpu; ++ ++ lockdep_assert_held(&rq->lock); ++ ++ if (test_tsk_need_resched(curr)) ++ return; ++ ++ cpu = cpu_of(rq); ++ ++ if (cpu == smp_processor_id()) { ++ set_tsk_need_resched(curr); ++ set_preempt_need_resched(); ++ return; ++ } ++ ++ if (set_nr_and_not_polling(curr)) ++ smp_send_reschedule(cpu); ++ else ++ trace_sched_wake_idle_without_ipi(cpu); ++} ++ ++void resched_cpu(int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ unsigned long flags; ++ ++ if (!raw_spin_trylock_irqsave(&rq->lock, flags)) ++ return; ++ resched_curr(rq); ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++} ++ ++#ifdef CONFIG_SMP ++#ifdef CONFIG_NO_HZ_COMMON ++/* ++ * In the semi idle case, use the nearest busy cpu for migrating timers ++ * from an idle cpu. This is good for power-savings. ++ * ++ * We don't do similar optimization for completely idle system, as ++ * selecting an idle cpu will add more delays to the timers than intended ++ * (as that cpu's timer base may not be uptodate wrt jiffies etc). ++ */ ++int get_nohz_timer_target(int pinned) ++{ ++ int cpu = smp_processor_id(); ++ int i; ++ struct sched_domain *sd; ++ ++ if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu)) ++ return cpu; ++ ++ rcu_read_lock(); ++ for_each_domain(cpu, sd) { ++ for_each_cpu(i, sched_domain_span(sd)) { ++ if (!idle_cpu(i)) { ++ cpu = i; ++ goto unlock; ++ } ++ } ++ } ++unlock: ++ rcu_read_unlock(); ++ return cpu; ++} ++/* ++ * When add_timer_on() enqueues a timer into the timer wheel of an ++ * idle CPU then this timer might expire before the next timer event ++ * which is scheduled to wake up that CPU. In case of a completely ++ * idle system the next event might even be infinite time into the ++ * future. wake_up_idle_cpu() ensures that the CPU is woken up and ++ * leaves the inner idle loop so the newly added timer is taken into ++ * account when the CPU goes back to idle and evaluates the timer ++ * wheel for the next timer event. ++ */ ++static void wake_up_idle_cpu(int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ ++ if (cpu == smp_processor_id()) ++ return; ++ ++ if (set_nr_and_not_polling(rq->idle)) ++ smp_send_reschedule(cpu); ++ else ++ trace_sched_wake_idle_without_ipi(cpu); ++} ++ ++static bool wake_up_full_nohz_cpu(int cpu) ++{ ++ /* ++ * We just need the target to call irq_exit() and re-evaluate ++ * the next tick. The nohz full kick at least implies that. ++ * If needed we can still optimize that later with an ++ * empty IRQ. ++ */ ++ if (tick_nohz_full_cpu(cpu)) { ++ if (cpu != smp_processor_id() || ++ tick_nohz_tick_stopped()) ++ tick_nohz_full_kick_cpu(cpu); ++ return true; ++ } ++ ++ return false; ++} ++ ++void wake_up_nohz_cpu(int cpu) ++{ ++ if (!wake_up_full_nohz_cpu(cpu)) ++ wake_up_idle_cpu(cpu); ++} ++ ++static inline bool got_nohz_idle_kick(void) ++{ ++ int cpu = smp_processor_id(); ++ ++ if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu))) ++ return false; ++ ++ if (idle_cpu(cpu) && !need_resched()) ++ return true; ++ ++ /* ++ * We can't run Idle Load Balance on this CPU for this time so we ++ * cancel it and clear NOHZ_BALANCE_KICK ++ */ ++ clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)); ++ return false; ++} ++ ++#else /* CONFIG_NO_HZ_COMMON */ ++ ++static inline bool got_nohz_idle_kick(void) ++{ ++ return false; ++} ++ ++#endif /* CONFIG_NO_HZ_COMMON */ ++ ++#ifdef CONFIG_NO_HZ_FULL ++bool sched_can_stop_tick(void) ++{ ++ /* ++ * FIFO realtime policy runs the highest priority task. Other runnable ++ * tasks are of a lower priority. The scheduler tick does nothing. ++ */ ++ if (current->policy == SCHED_FIFO) ++ return true; ++ ++ /* ++ * Round-robin realtime tasks time slice with other tasks at the same ++ * realtime priority. Is this task the only one at this priority? ++ */ ++ if (current->policy == SCHED_RR) { ++ struct sched_rt_entity *rt_se = ¤t->rt; ++ ++ return rt_se->run_list.prev == rt_se->run_list.next; ++ } ++ ++ /* ++ * More than one running task need preemption. ++ * nr_running update is assumed to be visible ++ * after IPI is sent from wakers. ++ */ ++ if (this_rq()->nr_running > 1) ++ return false; ++ ++ return true; ++} ++#endif /* CONFIG_NO_HZ_FULL */ ++ ++void sched_avg_update(struct rq *rq) ++{ ++ s64 period = sched_avg_period(); ++ ++ while ((s64)(rq_clock(rq) - rq->age_stamp) > period) { ++ /* ++ * Inline assembly required to prevent the compiler ++ * optimising this loop into a divmod call. ++ * See __iter_div_u64_rem() for another example of this. ++ */ ++ asm("" : "+rm" (rq->age_stamp)); ++ rq->age_stamp += period; ++ rq->rt_avg /= 2; ++ } ++} ++ ++#endif /* CONFIG_SMP */ ++ ++#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ ++ (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) ++/* ++ * Iterate task_group tree rooted at *from, calling @down when first entering a ++ * node and @up when leaving it for the final time. ++ * ++ * Caller must hold rcu_lock or sufficient equivalent. ++ */ ++int walk_tg_tree_from(struct task_group *from, ++ tg_visitor down, tg_visitor up, void *data) ++{ ++ struct task_group *parent, *child; ++ int ret; ++ ++ parent = from; ++ ++down: ++ ret = (*down)(parent, data); ++ if (ret) ++ goto out; ++ list_for_each_entry_rcu(child, &parent->children, siblings) { ++ parent = child; ++ goto down; ++ ++up: ++ continue; ++ } ++ ret = (*up)(parent, data); ++ if (ret || parent == from) ++ goto out; ++ ++ child = parent; ++ parent = parent->parent; ++ if (parent) ++ goto up; ++out: ++ return ret; ++} ++ ++int tg_nop(struct task_group *tg, void *data) ++{ ++ return 0; ++} ++#endif ++ ++static void set_load_weight(struct task_struct *p) ++{ ++ int prio = p->static_prio - MAX_RT_PRIO; ++ struct load_weight *load = &p->se.load; ++ ++ /* ++ * SCHED_IDLE tasks get minimal weight: ++ */ ++ if (p->policy == SCHED_IDLE) { ++ load->weight = scale_load(WEIGHT_IDLEPRIO); ++ load->inv_weight = WMULT_IDLEPRIO; ++ return; ++ } ++ ++ load->weight = scale_load(prio_to_weight[prio]); ++ load->inv_weight = prio_to_wmult[prio]; ++} ++ ++static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) ++{ ++ update_rq_clock(rq); ++ sched_info_queued(rq, p); ++ p->sched_class->enqueue_task(rq, p, flags); ++} ++ ++static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) ++{ ++ update_rq_clock(rq); ++ sched_info_dequeued(rq, p); ++ p->sched_class->dequeue_task(rq, p, flags); ++} ++ ++void activate_task(struct rq *rq, struct task_struct *p, int flags) ++{ ++ if (task_contributes_to_load(p)) ++ rq->nr_uninterruptible--; ++ ++ enqueue_task(rq, p, flags); ++} ++ ++void deactivate_task(struct rq *rq, struct task_struct *p, int flags) ++{ ++ if (task_contributes_to_load(p)) ++ rq->nr_uninterruptible++; ++ ++ dequeue_task(rq, p, flags); ++} ++ ++static void update_rq_clock_task(struct rq *rq, s64 delta) ++{ ++/* ++ * In theory, the compile should just see 0 here, and optimize out the call ++ * to sched_rt_avg_update. But I don't trust it... ++ */ ++#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) ++ s64 steal = 0, irq_delta = 0; ++#endif ++#ifdef CONFIG_IRQ_TIME_ACCOUNTING ++ irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; ++ ++ /* ++ * Since irq_time is only updated on {soft,}irq_exit, we might run into ++ * this case when a previous update_rq_clock() happened inside a ++ * {soft,}irq region. ++ * ++ * When this happens, we stop ->clock_task and only update the ++ * prev_irq_time stamp to account for the part that fit, so that a next ++ * update will consume the rest. This ensures ->clock_task is ++ * monotonic. ++ * ++ * It does however cause some slight miss-attribution of {soft,}irq ++ * time, a more accurate solution would be to update the irq_time using ++ * the current rq->clock timestamp, except that would require using ++ * atomic ops. ++ */ ++ if (irq_delta > delta) ++ irq_delta = delta; ++ ++ rq->prev_irq_time += irq_delta; ++ delta -= irq_delta; ++#endif ++#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING ++ if (static_key_false((¶virt_steal_rq_enabled))) { ++ steal = paravirt_steal_clock(cpu_of(rq)); ++ steal -= rq->prev_steal_time_rq; ++ ++ if (unlikely(steal > delta)) ++ steal = delta; ++ ++ rq->prev_steal_time_rq += steal; ++ delta -= steal; ++ } ++#endif ++ ++ rq->clock_task += delta; ++ ++#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) ++ if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) ++ sched_rt_avg_update(rq, irq_delta + steal); ++#endif ++} ++ ++void sched_set_stop_task(int cpu, struct task_struct *stop) ++{ ++ struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; ++ struct task_struct *old_stop = cpu_rq(cpu)->stop; ++ ++ if (stop) { ++ /* ++ * Make it appear like a SCHED_FIFO task, its something ++ * userspace knows about and won't get confused about. ++ * ++ * Also, it will make PI more or less work without too ++ * much confusion -- but then, stop work should not ++ * rely on PI working anyway. ++ */ ++ sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); ++ ++ stop->sched_class = &stop_sched_class; ++ } ++ ++ cpu_rq(cpu)->stop = stop; ++ ++ if (old_stop) { ++ /* ++ * Reset it back to a normal scheduling class so that ++ * it can die in pieces. ++ */ ++ old_stop->sched_class = &rt_sched_class; ++ } ++} ++ ++/* ++ * __normal_prio - return the priority that is based on the static prio ++ */ ++static inline int __normal_prio(struct task_struct *p) ++{ ++ return p->static_prio; ++} ++ ++/* ++ * Calculate the expected normal priority: i.e. priority ++ * without taking RT-inheritance into account. Might be ++ * boosted by interactivity modifiers. Changes upon fork, ++ * setprio syscalls, and whenever the interactivity ++ * estimator recalculates. ++ */ ++static inline int normal_prio(struct task_struct *p) ++{ ++ int prio; ++ ++ if (task_has_dl_policy(p)) ++ prio = MAX_DL_PRIO-1; ++ else if (task_has_rt_policy(p)) ++ prio = MAX_RT_PRIO-1 - p->rt_priority; ++ else ++ prio = __normal_prio(p); ++ return prio; ++} ++ ++/* ++ * Calculate the current priority, i.e. the priority ++ * taken into account by the scheduler. This value might ++ * be boosted by RT tasks, or might be boosted by ++ * interactivity modifiers. Will be RT if the task got ++ * RT-boosted. If not then it returns p->normal_prio. ++ */ ++static int effective_prio(struct task_struct *p) ++{ ++ p->normal_prio = normal_prio(p); ++ /* ++ * If we are RT tasks or we were boosted to RT priority, ++ * keep the priority unchanged. Otherwise, update priority ++ * to the normal priority: ++ */ ++ if (!rt_prio(p->prio)) ++ return p->normal_prio; ++ return p->prio; ++} ++ ++/** ++ * task_curr - is this task currently executing on a CPU? ++ * @p: the task in question. ++ * ++ * Return: 1 if the task is currently executing. 0 otherwise. ++ */ ++inline int task_curr(const struct task_struct *p) ++{ ++ return cpu_curr(task_cpu(p)) == p; ++} ++ ++/* ++ * Can drop rq->lock because from sched_class::switched_from() methods drop it. ++ */ ++static inline void check_class_changed(struct rq *rq, struct task_struct *p, ++ const struct sched_class *prev_class, ++ int oldprio) ++{ ++ if (prev_class != p->sched_class) { ++ if (prev_class->switched_from) ++ prev_class->switched_from(rq, p); ++ /* Possble rq->lock 'hole'. */ ++ p->sched_class->switched_to(rq, p); ++ } else if (oldprio != p->prio || dl_task(p)) ++ p->sched_class->prio_changed(rq, p, oldprio); ++} ++ ++void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) ++{ ++ const struct sched_class *class; ++ ++ if (p->sched_class == rq->curr->sched_class) { ++ rq->curr->sched_class->check_preempt_curr(rq, p, flags); ++ } else { ++ for_each_class(class) { ++ if (class == rq->curr->sched_class) ++ break; ++ if (class == p->sched_class) { ++ resched_curr(rq); ++ break; ++ } ++ } ++ } ++ ++ /* ++ * A queue event has occurred, and we're going to schedule. In ++ * this case, we can save a useless back to back clock update. ++ */ ++ if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) ++ rq_clock_skip_update(rq, true); ++} ++ ++#ifdef CONFIG_SMP ++void set_task_cpu(struct task_struct *p, unsigned int new_cpu) ++{ ++#ifdef CONFIG_SCHED_DEBUG ++ /* ++ * We should never call set_task_cpu() on a blocked task, ++ * ttwu() will sort out the placement. ++ */ ++ WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && ++ !p->on_rq); ++ ++#ifdef CONFIG_LOCKDEP ++ /* ++ * The caller should hold either p->pi_lock or rq->lock, when changing ++ * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. ++ * ++ * sched_move_task() holds both and thus holding either pins the cgroup, ++ * see task_group(). ++ * ++ * Furthermore, all task_rq users should acquire both locks, see ++ * task_rq_lock(). ++ */ ++ WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || ++ lockdep_is_held(&task_rq(p)->lock))); ++#endif ++#endif ++ ++ trace_sched_migrate_task(p, new_cpu); ++ ++ if (task_cpu(p) != new_cpu) { ++ if (p->sched_class->migrate_task_rq) ++ p->sched_class->migrate_task_rq(p, new_cpu); ++ p->se.nr_migrations++; ++ perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0); ++ } ++ ++ __set_task_cpu(p, new_cpu); ++} ++ ++static void __migrate_swap_task(struct task_struct *p, int cpu) ++{ ++ if (task_on_rq_queued(p)) { ++ struct rq *src_rq, *dst_rq; ++ ++ src_rq = task_rq(p); ++ dst_rq = cpu_rq(cpu); ++ ++ deactivate_task(src_rq, p, 0); ++ set_task_cpu(p, cpu); ++ activate_task(dst_rq, p, 0); ++ check_preempt_curr(dst_rq, p, 0); ++ } else { ++ /* ++ * Task isn't running anymore; make it appear like we migrated ++ * it before it went to sleep. This means on wakeup we make the ++ * previous cpu our targer instead of where it really is. ++ */ ++ p->wake_cpu = cpu; ++ } ++} ++ ++struct migration_swap_arg { ++ struct task_struct *src_task, *dst_task; ++ int src_cpu, dst_cpu; ++}; ++ ++static int migrate_swap_stop(void *data) ++{ ++ struct migration_swap_arg *arg = data; ++ struct rq *src_rq, *dst_rq; ++ int ret = -EAGAIN; ++ ++ src_rq = cpu_rq(arg->src_cpu); ++ dst_rq = cpu_rq(arg->dst_cpu); ++ ++ double_raw_lock(&arg->src_task->pi_lock, ++ &arg->dst_task->pi_lock); ++ double_rq_lock(src_rq, dst_rq); ++ if (task_cpu(arg->dst_task) != arg->dst_cpu) ++ goto unlock; ++ ++ if (task_cpu(arg->src_task) != arg->src_cpu) ++ goto unlock; ++ ++ if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task))) ++ goto unlock; ++ ++ if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task))) ++ goto unlock; ++ ++ __migrate_swap_task(arg->src_task, arg->dst_cpu); ++ __migrate_swap_task(arg->dst_task, arg->src_cpu); ++ ++ ret = 0; ++ ++unlock: ++ double_rq_unlock(src_rq, dst_rq); ++ raw_spin_unlock(&arg->dst_task->pi_lock); ++ raw_spin_unlock(&arg->src_task->pi_lock); ++ ++ return ret; ++} ++ ++/* ++ * Cross migrate two tasks ++ */ ++int migrate_swap(struct task_struct *cur, struct task_struct *p) ++{ ++ struct migration_swap_arg arg; ++ int ret = -EINVAL; ++ ++ arg = (struct migration_swap_arg){ ++ .src_task = cur, ++ .src_cpu = task_cpu(cur), ++ .dst_task = p, ++ .dst_cpu = task_cpu(p), ++ }; ++ ++ if (arg.src_cpu == arg.dst_cpu) ++ goto out; ++ ++ /* ++ * These three tests are all lockless; this is OK since all of them ++ * will be re-checked with proper locks held further down the line. ++ */ ++ if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) ++ goto out; ++ ++ if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task))) ++ goto out; ++ ++ if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task))) ++ goto out; ++ ++ trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); ++ ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); ++ ++out: ++ return ret; ++} ++ ++struct migration_arg { ++ struct task_struct *task; ++ int dest_cpu; ++}; ++ ++static int migration_cpu_stop(void *data); ++ ++/* ++ * wait_task_inactive - wait for a thread to unschedule. ++ * ++ * If @match_state is nonzero, it's the @p->state value just checked and ++ * not expected to change. If it changes, i.e. @p might have woken up, ++ * then return zero. When we succeed in waiting for @p to be off its CPU, ++ * we return a positive number (its total switch count). If a second call ++ * a short while later returns the same number, the caller can be sure that ++ * @p has remained unscheduled the whole time. ++ * ++ * The caller must ensure that the task *will* unschedule sometime soon, ++ * else this function might spin for a *long* time. This function can't ++ * be called with interrupts off, or it may introduce deadlock with ++ * smp_call_function() if an IPI is sent by the same process we are ++ * waiting to become inactive. ++ */ ++unsigned long wait_task_inactive(struct task_struct *p, long match_state) ++{ ++ unsigned long flags; ++ int running, queued; ++ unsigned long ncsw; ++ struct rq *rq; ++ ++ for (;;) { ++ /* ++ * We do the initial early heuristics without holding ++ * any task-queue locks at all. We'll only try to get ++ * the runqueue lock when things look like they will ++ * work out! ++ */ ++ rq = task_rq(p); ++ ++ /* ++ * If the task is actively running on another CPU ++ * still, just relax and busy-wait without holding ++ * any locks. ++ * ++ * NOTE! Since we don't hold any locks, it's not ++ * even sure that "rq" stays as the right runqueue! ++ * But we don't care, since "task_running()" will ++ * return false if the runqueue has changed and p ++ * is actually now running somewhere else! ++ */ ++ while (task_running(rq, p)) { ++ if (match_state && unlikely(p->state != match_state)) ++ return 0; ++ cpu_relax(); ++ } ++ ++ /* ++ * Ok, time to look more closely! We need the rq ++ * lock now, to be *sure*. If we're wrong, we'll ++ * just go back and repeat. ++ */ ++ rq = task_rq_lock(p, &flags); ++ trace_sched_wait_task(p); ++ running = task_running(rq, p); ++ queued = task_on_rq_queued(p); ++ ncsw = 0; ++ if (!match_state || p->state == match_state) ++ ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ ++ task_rq_unlock(rq, p, &flags); ++ ++ /* ++ * If it changed from the expected state, bail out now. ++ */ ++ if (unlikely(!ncsw)) ++ break; ++ ++ /* ++ * Was it really running after all now that we ++ * checked with the proper locks actually held? ++ * ++ * Oops. Go back and try again.. ++ */ ++ if (unlikely(running)) { ++ cpu_relax(); ++ continue; ++ } ++ ++ /* ++ * It's not enough that it's not actively running, ++ * it must be off the runqueue _entirely_, and not ++ * preempted! ++ * ++ * So if it was still runnable (but just not actively ++ * running right now), it's preempted, and we should ++ * yield - it could be a while. ++ */ ++ if (unlikely(queued)) { ++ ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ); ++ ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ schedule_hrtimeout(&to, HRTIMER_MODE_REL); ++ continue; ++ } ++ ++ /* ++ * Ahh, all good. It wasn't running, and it wasn't ++ * runnable, which means that it will never become ++ * running in the future either. We're all done! ++ */ ++ break; ++ } ++ ++ return ncsw; ++} ++ ++/*** ++ * kick_process - kick a running thread to enter/exit the kernel ++ * @p: the to-be-kicked thread ++ * ++ * Cause a process which is running on another CPU to enter ++ * kernel-mode, without any delay. (to get signals handled.) ++ * ++ * NOTE: this function doesn't have to take the runqueue lock, ++ * because all it wants to ensure is that the remote task enters ++ * the kernel. If the IPI races and the task has been migrated ++ * to another CPU then no harm is done and the purpose has been ++ * achieved as well. ++ */ ++void kick_process(struct task_struct *p) ++{ ++ int cpu; ++ ++ preempt_disable(); ++ cpu = task_cpu(p); ++ if ((cpu != smp_processor_id()) && task_curr(p)) ++ smp_send_reschedule(cpu); ++ preempt_enable(); ++} ++EXPORT_SYMBOL_GPL(kick_process); ++#endif /* CONFIG_SMP */ ++ ++#ifdef CONFIG_SMP ++/* ++ * ->cpus_allowed is protected by both rq->lock and p->pi_lock ++ */ ++static int select_fallback_rq(int cpu, struct task_struct *p) ++{ ++ int nid = cpu_to_node(cpu); ++ const struct cpumask *nodemask = NULL; ++ enum { cpuset, possible, fail } state = cpuset; ++ int dest_cpu; ++ ++ /* ++ * If the node that the cpu is on has been offlined, cpu_to_node() ++ * will return -1. There is no cpu on the node, and we should ++ * select the cpu on the other node. ++ */ ++ if (nid != -1) { ++ nodemask = cpumask_of_node(nid); ++ ++ /* Look for allowed, online CPU in same node. */ ++ for_each_cpu(dest_cpu, nodemask) { ++ if (!cpu_online(dest_cpu)) ++ continue; ++ if (!cpu_active(dest_cpu)) ++ continue; ++ if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) ++ return dest_cpu; ++ } ++ } ++ ++ for (;;) { ++ /* Any allowed, online CPU? */ ++ for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) { ++ if (!cpu_online(dest_cpu)) ++ continue; ++ if (!cpu_active(dest_cpu)) ++ continue; ++ goto out; ++ } ++ ++ switch (state) { ++ case cpuset: ++ /* No more Mr. Nice Guy. */ ++ cpuset_cpus_allowed_fallback(p); ++ state = possible; ++ break; ++ ++ case possible: ++ do_set_cpus_allowed(p, cpu_possible_mask); ++ state = fail; ++ break; ++ ++ case fail: ++ BUG(); ++ break; ++ } ++ } ++ ++out: ++ if (state != cpuset) { ++ /* ++ * Don't tell them about moving exiting tasks or ++ * kernel threads (both mm NULL), since they never ++ * leave kernel. ++ */ ++ if (p->mm && printk_ratelimit()) { ++ printk_deferred("process %d (%s) no longer affine to cpu%d\n", ++ task_pid_nr(p), p->comm, cpu); ++ } ++ } ++ ++ return dest_cpu; ++} ++ ++/* ++ * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. ++ */ ++static inline ++int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) ++{ ++ if (p->nr_cpus_allowed > 1) ++ cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags); ++ ++ /* ++ * In order not to call set_task_cpu() on a blocking task we need ++ * to rely on ttwu() to place the task on a valid ->cpus_allowed ++ * cpu. ++ * ++ * Since this is common to all placement strategies, this lives here. ++ * ++ * [ this allows ->select_task() to simply return task_cpu(p) and ++ * not worry about this generic constraint ] ++ */ ++ if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) || ++ !cpu_online(cpu))) ++ cpu = select_fallback_rq(task_cpu(p), p); ++ ++ return cpu; ++} ++ ++static void update_avg(u64 *avg, u64 sample) ++{ ++ s64 diff = sample - *avg; ++ *avg += diff >> 3; ++} ++#endif ++ ++static void ++ttwu_stat(struct task_struct *p, int cpu, int wake_flags) ++{ ++#ifdef CONFIG_SCHEDSTATS ++ struct rq *rq = this_rq(); ++ ++#ifdef CONFIG_SMP ++ int this_cpu = smp_processor_id(); ++ ++ if (cpu == this_cpu) { ++ schedstat_inc(rq, ttwu_local); ++ schedstat_inc(p, se.statistics.nr_wakeups_local); ++ } else { ++ struct sched_domain *sd; ++ ++ schedstat_inc(p, se.statistics.nr_wakeups_remote); ++ rcu_read_lock(); ++ for_each_domain(this_cpu, sd) { ++ if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { ++ schedstat_inc(sd, ttwu_wake_remote); ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ } ++ ++ if (wake_flags & WF_MIGRATED) ++ schedstat_inc(p, se.statistics.nr_wakeups_migrate); ++ ++#endif /* CONFIG_SMP */ ++ ++ schedstat_inc(rq, ttwu_count); ++ schedstat_inc(p, se.statistics.nr_wakeups); ++ ++ if (wake_flags & WF_SYNC) ++ schedstat_inc(p, se.statistics.nr_wakeups_sync); ++ ++#endif /* CONFIG_SCHEDSTATS */ ++} ++ ++static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) ++{ ++ activate_task(rq, p, en_flags); ++ p->on_rq = TASK_ON_RQ_QUEUED; ++ ++ /* if a worker is waking up, notify workqueue */ ++ if (p->flags & PF_WQ_WORKER) ++ wq_worker_waking_up(p, cpu_of(rq)); ++} ++ ++/* ++ * Mark the task runnable and perform wakeup-preemption. ++ */ ++static void ++ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) ++{ ++ check_preempt_curr(rq, p, wake_flags); ++ trace_sched_wakeup(p, true); ++ ++ p->state = TASK_RUNNING; ++#ifdef CONFIG_SMP ++ if (p->sched_class->task_woken) ++ p->sched_class->task_woken(rq, p); ++ ++ if (rq->idle_stamp) { ++ u64 delta = rq_clock(rq) - rq->idle_stamp; ++ u64 max = 2*rq->max_idle_balance_cost; ++ ++ update_avg(&rq->avg_idle, delta); ++ ++ if (rq->avg_idle > max) ++ rq->avg_idle = max; ++ ++ rq->idle_stamp = 0; ++ } ++#endif ++} ++ ++static void ++ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) ++{ ++#ifdef CONFIG_SMP ++ if (p->sched_contributes_to_load) ++ rq->nr_uninterruptible--; ++#endif ++ ++ ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING); ++ ttwu_do_wakeup(rq, p, wake_flags); ++} ++ ++/* ++ * Called in case the task @p isn't fully descheduled from its runqueue, ++ * in this case we must do a remote wakeup. Its a 'light' wakeup though, ++ * since all we need to do is flip p->state to TASK_RUNNING, since ++ * the task is still ->on_rq. ++ */ ++static int ttwu_remote(struct task_struct *p, int wake_flags) ++{ ++ struct rq *rq; ++ int ret = 0; ++ ++ rq = __task_rq_lock(p); ++ if (task_on_rq_queued(p)) { ++ /* check_preempt_curr() may use rq clock */ ++ update_rq_clock(rq); ++ ttwu_do_wakeup(rq, p, wake_flags); ++ ret = 1; ++ } ++ __task_rq_unlock(rq); ++ ++ return ret; ++} ++ ++#ifdef CONFIG_SMP ++void sched_ttwu_pending(void) ++{ ++ struct rq *rq = this_rq(); ++ struct llist_node *llist = llist_del_all(&rq->wake_list); ++ struct task_struct *p; ++ unsigned long flags; ++ ++ if (!llist) ++ return; ++ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ ++ while (llist) { ++ p = llist_entry(llist, struct task_struct, wake_entry); ++ llist = llist_next(llist); ++ ttwu_do_activate(rq, p, 0); ++ } ++ ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++} ++ ++void scheduler_ipi(void) ++{ ++ /* ++ * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting ++ * TIF_NEED_RESCHED remotely (for the first time) will also send ++ * this IPI. ++ */ ++ preempt_fold_need_resched(); ++ ++ if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) ++ return; ++ ++ /* ++ * Not all reschedule IPI handlers call irq_enter/irq_exit, since ++ * traditionally all their work was done from the interrupt return ++ * path. Now that we actually do some work, we need to make sure ++ * we do call them. ++ * ++ * Some archs already do call them, luckily irq_enter/exit nest ++ * properly. ++ * ++ * Arguably we should visit all archs and update all handlers, ++ * however a fair share of IPIs are still resched only so this would ++ * somewhat pessimize the simple resched case. ++ */ ++ irq_enter(); ++ sched_ttwu_pending(); ++ ++ /* ++ * Check if someone kicked us for doing the nohz idle load balance. ++ */ ++ if (unlikely(got_nohz_idle_kick())) { ++ this_rq()->idle_balance = 1; ++ raise_softirq_irqoff(SCHED_SOFTIRQ); ++ } ++ irq_exit(); ++} ++ ++static void ttwu_queue_remote(struct task_struct *p, int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ ++ if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { ++ if (!set_nr_if_polling(rq->idle)) ++ smp_send_reschedule(cpu); ++ else ++ trace_sched_wake_idle_without_ipi(cpu); ++ } ++} ++ ++void wake_up_if_idle(int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ unsigned long flags; ++ ++ rcu_read_lock(); ++ ++ if (!is_idle_task(rcu_dereference(rq->curr))) ++ goto out; ++ ++ if (set_nr_if_polling(rq->idle)) { ++ trace_sched_wake_idle_without_ipi(cpu); ++ } else { ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ if (is_idle_task(rq->curr)) ++ smp_send_reschedule(cpu); ++ /* Else cpu is not in idle, do nothing here */ ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ } ++ ++out: ++ rcu_read_unlock(); ++} ++ ++bool cpus_share_cache(int this_cpu, int that_cpu) ++{ ++ return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); ++} ++#endif /* CONFIG_SMP */ ++ ++static void ttwu_queue(struct task_struct *p, int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ ++#if defined(CONFIG_SMP) ++ if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { ++ sched_clock_cpu(cpu); /* sync clocks x-cpu */ ++ ttwu_queue_remote(p, cpu); ++ return; ++ } ++#endif ++ ++ raw_spin_lock(&rq->lock); ++ ttwu_do_activate(rq, p, 0); ++ raw_spin_unlock(&rq->lock); ++} ++ ++/** ++ * try_to_wake_up - wake up a thread ++ * @p: the thread to be awakened ++ * @state: the mask of task states that can be woken ++ * @wake_flags: wake modifier flags (WF_*) ++ * ++ * Put it on the run-queue if it's not already there. The "current" ++ * thread is always on the run-queue (except when the actual ++ * re-schedule is in progress), and as such you're allowed to do ++ * the simpler "current->state = TASK_RUNNING" to mark yourself ++ * runnable without the overhead of this. ++ * ++ * Return: %true if @p was woken up, %false if it was already running. ++ * or @state didn't match @p's state. ++ */ ++static int ++try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) ++{ ++ unsigned long flags; ++ int cpu, success = 0; ++ ++ /* ++ * If we are going to wake up a thread waiting for CONDITION we ++ * need to ensure that CONDITION=1 done by the caller can not be ++ * reordered with p->state check below. This pairs with mb() in ++ * set_current_state() the waiting thread does. ++ */ ++ smp_mb__before_spinlock(); ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++ if (!(p->state & state)) ++ goto out; ++ ++ success = 1; /* we're going to change ->state */ ++ cpu = task_cpu(p); ++ ++ if (p->on_rq && ttwu_remote(p, wake_flags)) ++ goto stat; ++ ++#ifdef CONFIG_SMP ++ /* ++ * If the owning (remote) cpu is still in the middle of schedule() with ++ * this task as prev, wait until its done referencing the task. ++ */ ++ while (p->on_cpu) ++ cpu_relax(); ++ /* ++ * Pairs with the smp_wmb() in finish_lock_switch(). ++ */ ++ smp_rmb(); ++ ++ p->sched_contributes_to_load = !!task_contributes_to_load(p); ++ p->state = TASK_WAKING; ++ ++ if (p->sched_class->task_waking) ++ p->sched_class->task_waking(p); ++ ++ cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); ++ if (task_cpu(p) != cpu) { ++ wake_flags |= WF_MIGRATED; ++ set_task_cpu(p, cpu); ++ } ++#endif /* CONFIG_SMP */ ++ ++ ttwu_queue(p, cpu); ++stat: ++ ttwu_stat(p, cpu, wake_flags); ++out: ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++ ++ return success; ++} ++ ++/** ++ * try_to_wake_up_local - try to wake up a local task with rq lock held ++ * @p: the thread to be awakened ++ * ++ * Put @p on the run-queue if it's not already there. The caller must ++ * ensure that this_rq() is locked, @p is bound to this_rq() and not ++ * the current task. ++ */ ++static void try_to_wake_up_local(struct task_struct *p) ++{ ++ struct rq *rq = task_rq(p); ++ ++ if (WARN_ON_ONCE(rq != this_rq()) || ++ WARN_ON_ONCE(p == current)) ++ return; ++ ++ lockdep_assert_held(&rq->lock); ++ ++ if (!raw_spin_trylock(&p->pi_lock)) { ++ raw_spin_unlock(&rq->lock); ++ raw_spin_lock(&p->pi_lock); ++ raw_spin_lock(&rq->lock); ++ } ++ ++ if (!(p->state & TASK_NORMAL)) ++ goto out; ++ ++ if (!task_on_rq_queued(p)) ++ ttwu_activate(rq, p, ENQUEUE_WAKEUP); ++ ++ ttwu_do_wakeup(rq, p, 0); ++ ttwu_stat(p, smp_processor_id(), 0); ++out: ++ raw_spin_unlock(&p->pi_lock); ++} ++ ++/** ++ * wake_up_process - Wake up a specific process ++ * @p: The process to be woken up. ++ * ++ * Attempt to wake up the nominated process and move it to the set of runnable ++ * processes. ++ * ++ * Return: 1 if the process was woken up, 0 if it was already running. ++ * ++ * It may be assumed that this function implies a write memory barrier before ++ * changing the task state if and only if any tasks are woken up. ++ */ ++int wake_up_process(struct task_struct *p) ++{ ++ WARN_ON(task_is_stopped_or_traced(p)); ++ return try_to_wake_up(p, TASK_NORMAL, 0); ++} ++EXPORT_SYMBOL(wake_up_process); ++ ++int wake_up_state(struct task_struct *p, unsigned int state) ++{ ++ return try_to_wake_up(p, state, 0); ++} ++ ++/* ++ * This function clears the sched_dl_entity static params. ++ */ ++void __dl_clear_params(struct task_struct *p) ++{ ++ struct sched_dl_entity *dl_se = &p->dl; ++ ++ dl_se->dl_runtime = 0; ++ dl_se->dl_deadline = 0; ++ dl_se->dl_period = 0; ++ dl_se->flags = 0; ++ dl_se->dl_bw = 0; ++ ++ dl_se->dl_throttled = 0; ++ dl_se->dl_new = 1; ++ dl_se->dl_yielded = 0; ++} ++ ++/* ++ * Perform scheduler related setup for a newly forked process p. ++ * p is forked by current. ++ * ++ * __sched_fork() is basic setup used by init_idle() too: ++ */ ++static void __sched_fork(unsigned long clone_flags, struct task_struct *p) ++{ ++ p->on_rq = 0; ++ ++ p->se.on_rq = 0; ++ p->se.exec_start = 0; ++ p->se.sum_exec_runtime = 0; ++ p->se.prev_sum_exec_runtime = 0; ++ p->se.nr_migrations = 0; ++ p->se.vruntime = 0; ++#ifdef CONFIG_SMP ++ p->se.avg.decay_count = 0; ++#endif ++ INIT_LIST_HEAD(&p->se.group_node); ++ ++#ifdef CONFIG_SCHEDSTATS ++ memset(&p->se.statistics, 0, sizeof(p->se.statistics)); ++#endif ++ ++ RB_CLEAR_NODE(&p->dl.rb_node); ++ init_dl_task_timer(&p->dl); ++ __dl_clear_params(p); ++ ++ INIT_LIST_HEAD(&p->rt.run_list); ++ ++#ifdef CONFIG_PREEMPT_NOTIFIERS ++ INIT_HLIST_HEAD(&p->preempt_notifiers); ++#endif ++ ++#ifdef CONFIG_NUMA_BALANCING ++ if (p->mm && atomic_read(&p->mm->mm_users) == 1) { ++ p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); ++ p->mm->numa_scan_seq = 0; ++ } ++ ++ if (clone_flags & CLONE_VM) ++ p->numa_preferred_nid = current->numa_preferred_nid; ++ else ++ p->numa_preferred_nid = -1; ++ ++ p->node_stamp = 0ULL; ++ p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0; ++ p->numa_scan_period = sysctl_numa_balancing_scan_delay; ++ p->numa_work.next = &p->numa_work; ++ p->numa_faults = NULL; ++ p->last_task_numa_placement = 0; ++ p->last_sum_exec_runtime = 0; ++ ++ p->numa_group = NULL; ++#endif /* CONFIG_NUMA_BALANCING */ ++} ++ ++#ifdef CONFIG_NUMA_BALANCING ++#ifdef CONFIG_SCHED_DEBUG ++void set_numabalancing_state(bool enabled) ++{ ++ if (enabled) ++ sched_feat_set("NUMA"); ++ else ++ sched_feat_set("NO_NUMA"); ++} ++#else ++__read_mostly bool numabalancing_enabled; ++ ++void set_numabalancing_state(bool enabled) ++{ ++ numabalancing_enabled = enabled; ++} ++#endif /* CONFIG_SCHED_DEBUG */ ++ ++#ifdef CONFIG_PROC_SYSCTL ++int sysctl_numa_balancing(struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, loff_t *ppos) ++{ ++ struct ctl_table t; ++ int err; ++ int state = numabalancing_enabled; ++ ++ if (write && !capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ t = *table; ++ t.data = &state; ++ err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); ++ if (err < 0) ++ return err; ++ if (write) ++ set_numabalancing_state(state); ++ return err; ++} ++#endif ++#endif ++ ++/* ++ * fork()/clone()-time setup: ++ */ ++int sched_fork(unsigned long clone_flags, struct task_struct *p) ++{ ++ unsigned long flags; ++ int cpu = get_cpu(); ++ ++ __sched_fork(clone_flags, p); ++ /* ++ * We mark the process as running here. This guarantees that ++ * nobody will actually run it, and a signal or other external ++ * event cannot wake it up and insert it on the runqueue either. ++ */ ++ p->state = TASK_RUNNING; ++ ++ /* ++ * Make sure we do not leak PI boosting priority to the child. ++ */ ++ p->prio = current->normal_prio; ++ ++ /* ++ * Revert to default priority/policy on fork if requested. ++ */ ++ if (unlikely(p->sched_reset_on_fork)) { ++ if (task_has_dl_policy(p) || task_has_rt_policy(p)) { ++ p->policy = SCHED_NORMAL; ++ p->static_prio = NICE_TO_PRIO(0); ++ p->rt_priority = 0; ++ } else if (PRIO_TO_NICE(p->static_prio) < 0) ++ p->static_prio = NICE_TO_PRIO(0); ++ ++ p->prio = p->normal_prio = __normal_prio(p); ++ set_load_weight(p); ++ ++ /* ++ * We don't need the reset flag anymore after the fork. It has ++ * fulfilled its duty: ++ */ ++ p->sched_reset_on_fork = 0; ++ } ++ ++ if (dl_prio(p->prio)) { ++ put_cpu(); ++ return -EAGAIN; ++ } else if (rt_prio(p->prio)) { ++ p->sched_class = &rt_sched_class; ++ } else { ++ p->sched_class = &fair_sched_class; ++ } ++ ++ if (p->sched_class->task_fork) ++ p->sched_class->task_fork(p); ++ ++ /* ++ * The child is not yet in the pid-hash so no cgroup attach races, ++ * and the cgroup is pinned to this child due to cgroup_fork() ++ * is ran before sched_fork(). ++ * ++ * Silence PROVE_RCU. ++ */ ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++ set_task_cpu(p, cpu); ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++ ++#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) ++ if (likely(sched_info_on())) ++ memset(&p->sched_info, 0, sizeof(p->sched_info)); ++#endif ++#if defined(CONFIG_SMP) ++ p->on_cpu = 0; ++#endif ++ init_task_preempt_count(p); ++#ifdef CONFIG_SMP ++ plist_node_init(&p->pushable_tasks, MAX_PRIO); ++ RB_CLEAR_NODE(&p->pushable_dl_tasks); ++#endif ++ ++ put_cpu(); ++ return 0; ++} ++ ++unsigned long to_ratio(u64 period, u64 runtime) ++{ ++ if (runtime == RUNTIME_INF) ++ return 1ULL << 20; ++ ++ /* ++ * Doing this here saves a lot of checks in all ++ * the calling paths, and returning zero seems ++ * safe for them anyway. ++ */ ++ if (period == 0) ++ return 0; ++ ++ return div64_u64(runtime << 20, period); ++} ++ ++#ifdef CONFIG_SMP ++inline struct dl_bw *dl_bw_of(int i) ++{ ++ rcu_lockdep_assert(rcu_read_lock_sched_held(), ++ "sched RCU must be held"); ++ return &cpu_rq(i)->rd->dl_bw; ++} ++ ++static inline int dl_bw_cpus(int i) ++{ ++ struct root_domain *rd = cpu_rq(i)->rd; ++ int cpus = 0; ++ ++ rcu_lockdep_assert(rcu_read_lock_sched_held(), ++ "sched RCU must be held"); ++ for_each_cpu_and(i, rd->span, cpu_active_mask) ++ cpus++; ++ ++ return cpus; ++} ++#else ++inline struct dl_bw *dl_bw_of(int i) ++{ ++ return &cpu_rq(i)->dl.dl_bw; ++} ++ ++static inline int dl_bw_cpus(int i) ++{ ++ return 1; ++} ++#endif ++ ++/* ++ * We must be sure that accepting a new task (or allowing changing the ++ * parameters of an existing one) is consistent with the bandwidth ++ * constraints. If yes, this function also accordingly updates the currently ++ * allocated bandwidth to reflect the new situation. ++ * ++ * This function is called while holding p's rq->lock. ++ * ++ * XXX we should delay bw change until the task's 0-lag point, see ++ * __setparam_dl(). ++ */ ++static int dl_overflow(struct task_struct *p, int policy, ++ const struct sched_attr *attr) ++{ ++ ++ struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); ++ u64 period = attr->sched_period ?: attr->sched_deadline; ++ u64 runtime = attr->sched_runtime; ++ u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0; ++ int cpus, err = -1; ++ ++ if (new_bw == p->dl.dl_bw) ++ return 0; ++ ++ /* ++ * Either if a task, enters, leave, or stays -deadline but changes ++ * its parameters, we may need to update accordingly the total ++ * allocated bandwidth of the container. ++ */ ++ raw_spin_lock(&dl_b->lock); ++ cpus = dl_bw_cpus(task_cpu(p)); ++ if (dl_policy(policy) && !task_has_dl_policy(p) && ++ !__dl_overflow(dl_b, cpus, 0, new_bw)) { ++ __dl_add(dl_b, new_bw); ++ err = 0; ++ } else if (dl_policy(policy) && task_has_dl_policy(p) && ++ !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) { ++ __dl_clear(dl_b, p->dl.dl_bw); ++ __dl_add(dl_b, new_bw); ++ err = 0; ++ } else if (!dl_policy(policy) && task_has_dl_policy(p)) { ++ __dl_clear(dl_b, p->dl.dl_bw); ++ err = 0; ++ } ++ raw_spin_unlock(&dl_b->lock); ++ ++ return err; ++} ++ ++extern void init_dl_bw(struct dl_bw *dl_b); ++ ++/* ++ * wake_up_new_task - wake up a newly created task for the first time. ++ * ++ * This function will do some initial scheduler statistics housekeeping ++ * that must be done for every newly created context, then puts the task ++ * on the runqueue and wakes it. ++ */ ++void wake_up_new_task(struct task_struct *p) ++{ ++ unsigned long flags; ++ struct rq *rq; ++ ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++#ifdef CONFIG_SMP ++ /* ++ * Fork balancing, do it here and not earlier because: ++ * - cpus_allowed can change in the fork path ++ * - any previously selected cpu might disappear through hotplug ++ */ ++ set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0)); ++#endif ++ ++ /* Initialize new task's runnable average */ ++ init_task_runnable_average(p); ++ rq = __task_rq_lock(p); ++ activate_task(rq, p, 0); ++ p->on_rq = TASK_ON_RQ_QUEUED; ++ trace_sched_wakeup_new(p, true); ++ check_preempt_curr(rq, p, WF_FORK); ++#ifdef CONFIG_SMP ++ if (p->sched_class->task_woken) ++ p->sched_class->task_woken(rq, p); ++#endif ++ task_rq_unlock(rq, p, &flags); ++} ++ ++#ifdef CONFIG_PREEMPT_NOTIFIERS ++ ++/** ++ * preempt_notifier_register - tell me when current is being preempted & rescheduled ++ * @notifier: notifier struct to register ++ */ ++void preempt_notifier_register(struct preempt_notifier *notifier) ++{ ++ hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); ++} ++EXPORT_SYMBOL_GPL(preempt_notifier_register); ++ ++/** ++ * preempt_notifier_unregister - no longer interested in preemption notifications ++ * @notifier: notifier struct to unregister ++ * ++ * This is safe to call from within a preemption notifier. ++ */ ++void preempt_notifier_unregister(struct preempt_notifier *notifier) ++{ ++ hlist_del(¬ifier->link); ++} ++EXPORT_SYMBOL_GPL(preempt_notifier_unregister); ++ ++static void fire_sched_in_preempt_notifiers(struct task_struct *curr) ++{ ++ struct preempt_notifier *notifier; ++ ++ hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) ++ notifier->ops->sched_in(notifier, raw_smp_processor_id()); ++} ++ ++static void ++fire_sched_out_preempt_notifiers(struct task_struct *curr, ++ struct task_struct *next) ++{ ++ struct preempt_notifier *notifier; ++ ++ hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) ++ notifier->ops->sched_out(notifier, next); ++} ++ ++#else /* !CONFIG_PREEMPT_NOTIFIERS */ ++ ++static void fire_sched_in_preempt_notifiers(struct task_struct *curr) ++{ ++} ++ ++static void ++fire_sched_out_preempt_notifiers(struct task_struct *curr, ++ struct task_struct *next) ++{ ++} ++ ++#endif /* CONFIG_PREEMPT_NOTIFIERS */ ++ ++/** ++ * prepare_task_switch - prepare to switch tasks ++ * @rq: the runqueue preparing to switch ++ * @prev: the current task that is being switched out ++ * @next: the task we are going to switch to. ++ * ++ * This is called with the rq lock held and interrupts off. It must ++ * be paired with a subsequent finish_task_switch after the context ++ * switch. ++ * ++ * prepare_task_switch sets up locking and calls architecture specific ++ * hooks. ++ */ ++static inline void ++prepare_task_switch(struct rq *rq, struct task_struct *prev, ++ struct task_struct *next) ++{ ++ trace_sched_switch(prev, next); ++ sched_info_switch(rq, prev, next); ++ perf_event_task_sched_out(prev, next); ++ fire_sched_out_preempt_notifiers(prev, next); ++ prepare_lock_switch(rq, next); ++ prepare_arch_switch(next); ++} ++ ++/** ++ * finish_task_switch - clean up after a task-switch ++ * @prev: the thread we just switched away from. ++ * ++ * finish_task_switch must be called after the context switch, paired ++ * with a prepare_task_switch call before the context switch. ++ * finish_task_switch will reconcile locking set up by prepare_task_switch, ++ * and do any other architecture-specific cleanup actions. ++ * ++ * Note that we may have delayed dropping an mm in context_switch(). If ++ * so, we finish that here outside of the runqueue lock. (Doing it ++ * with the lock held can cause deadlocks; see schedule() for ++ * details.) ++ * ++ * The context switch have flipped the stack from under us and restored the ++ * local variables which were saved when this task called schedule() in the ++ * past. prev == current is still correct but we need to recalculate this_rq ++ * because prev may have moved to another CPU. ++ */ ++static struct rq *finish_task_switch(struct task_struct *prev) ++ __releases(rq->lock) ++{ ++ struct rq *rq = this_rq(); ++ struct mm_struct *mm = rq->prev_mm; ++ long prev_state; ++ ++ rq->prev_mm = NULL; ++ ++ /* ++ * A task struct has one reference for the use as "current". ++ * If a task dies, then it sets TASK_DEAD in tsk->state and calls ++ * schedule one last time. The schedule call will never return, and ++ * the scheduled task must drop that reference. ++ * The test for TASK_DEAD must occur while the runqueue locks are ++ * still held, otherwise prev could be scheduled on another cpu, die ++ * there before we look at prev->state, and then the reference would ++ * be dropped twice. ++ * Manfred Spraul <manfred@colorfullife.com> ++ */ ++ prev_state = prev->state; ++ vtime_task_switch(prev); ++ finish_arch_switch(prev); ++ perf_event_task_sched_in(prev, current); ++ finish_lock_switch(rq, prev); ++ finish_arch_post_lock_switch(); ++ ++ fire_sched_in_preempt_notifiers(current); ++ if (mm) ++ mmdrop(mm); ++ if (unlikely(prev_state == TASK_DEAD)) { ++ if (prev->sched_class->task_dead) ++ prev->sched_class->task_dead(prev); ++ ++ /* ++ * Remove function-return probe instances associated with this ++ * task and put them back on the free list. ++ */ ++ kprobe_flush_task(prev); ++ put_task_struct(prev); ++ } ++ ++ tick_nohz_task_switch(current); ++ return rq; ++} ++ ++#ifdef CONFIG_SMP ++ ++/* rq->lock is NOT held, but preemption is disabled */ ++static inline void post_schedule(struct rq *rq) ++{ ++ if (rq->post_schedule) { ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ if (rq->curr->sched_class->post_schedule) ++ rq->curr->sched_class->post_schedule(rq); ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ ++ rq->post_schedule = 0; ++ } ++} ++ ++#else ++ ++static inline void post_schedule(struct rq *rq) ++{ ++} ++ ++#endif ++ ++/** ++ * schedule_tail - first thing a freshly forked thread must call. ++ * @prev: the thread we just switched away from. ++ */ ++asmlinkage __visible void schedule_tail(struct task_struct *prev) ++ __releases(rq->lock) ++{ ++ struct rq *rq; ++ ++ /* finish_task_switch() drops rq->lock and enables preemtion */ ++ preempt_disable(); ++ rq = finish_task_switch(prev); ++ post_schedule(rq); ++ preempt_enable(); ++ ++ if (current->set_child_tid) ++ put_user(task_pid_vnr(current), current->set_child_tid); ++} ++ ++/* ++ * context_switch - switch to the new MM and the new thread's register state. ++ */ ++static inline struct rq * ++context_switch(struct rq *rq, struct task_struct *prev, ++ struct task_struct *next) ++{ ++ struct mm_struct *mm, *oldmm; ++ ++ prepare_task_switch(rq, prev, next); ++ ++ mm = next->mm; ++ oldmm = prev->active_mm; ++ /* ++ * For paravirt, this is coupled with an exit in switch_to to ++ * combine the page table reload and the switch backend into ++ * one hypercall. ++ */ ++ arch_start_context_switch(prev); ++ ++ if (!mm) { ++ next->active_mm = oldmm; ++ atomic_inc(&oldmm->mm_count); ++ enter_lazy_tlb(oldmm, next); ++ } else ++ switch_mm(oldmm, mm, next); ++ ++ if (!prev->mm) { ++ prev->active_mm = NULL; ++ rq->prev_mm = oldmm; ++ } ++ /* ++ * Since the runqueue lock will be released by the next ++ * task (which is an invalid locking op but in the case ++ * of the scheduler it's an obvious special-case), so we ++ * do an early lockdep release here: ++ */ ++ spin_release(&rq->lock.dep_map, 1, _THIS_IP_); ++ ++ context_tracking_task_switch(prev, next); ++ /* Here we just switch the register state and the stack. */ ++ switch_to(prev, next, prev); ++ barrier(); ++ ++ return finish_task_switch(prev); ++} ++ ++/* ++ * nr_running and nr_context_switches: ++ * ++ * externally visible scheduler statistics: current number of runnable ++ * threads, total number of context switches performed since bootup. ++ */ ++unsigned long nr_running(void) ++{ ++ unsigned long i, sum = 0; ++ ++ for_each_online_cpu(i) ++ sum += cpu_rq(i)->nr_running; ++ ++ return sum; ++} ++ ++/* ++ * Check if only the current task is running on the cpu. ++ */ ++bool single_task_running(void) ++{ ++ if (cpu_rq(smp_processor_id())->nr_running == 1) ++ return true; ++ else ++ return false; ++} ++EXPORT_SYMBOL(single_task_running); ++ ++unsigned long long nr_context_switches(void) ++{ ++ int i; ++ unsigned long long sum = 0; ++ ++ for_each_possible_cpu(i) ++ sum += cpu_rq(i)->nr_switches; ++ ++ return sum; ++} ++ ++unsigned long nr_iowait(void) ++{ ++ unsigned long i, sum = 0; ++ ++ for_each_possible_cpu(i) ++ sum += atomic_read(&cpu_rq(i)->nr_iowait); ++ ++ return sum; ++} ++ ++unsigned long nr_iowait_cpu(int cpu) ++{ ++ struct rq *this = cpu_rq(cpu); ++ return atomic_read(&this->nr_iowait); ++} ++ ++void get_iowait_load(unsigned long *nr_waiters, unsigned long *load) ++{ ++ struct rq *this = this_rq(); ++ *nr_waiters = atomic_read(&this->nr_iowait); ++ *load = this->cpu_load[0]; ++} ++ ++#ifdef CONFIG_SMP ++ ++/* ++ * sched_exec - execve() is a valuable balancing opportunity, because at ++ * this point the task has the smallest effective memory and cache footprint. ++ */ ++void sched_exec(void) ++{ ++ struct task_struct *p = current; ++ unsigned long flags; ++ int dest_cpu; ++ ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++ dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0); ++ if (dest_cpu == smp_processor_id()) ++ goto unlock; ++ ++ if (likely(cpu_active(dest_cpu))) { ++ struct migration_arg arg = { p, dest_cpu }; ++ ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++ stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); ++ return; ++ } ++unlock: ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++} ++ ++#endif ++ ++DEFINE_PER_CPU(struct kernel_stat, kstat); ++DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); ++ ++EXPORT_PER_CPU_SYMBOL(kstat); ++EXPORT_PER_CPU_SYMBOL(kernel_cpustat); ++ ++/* ++ * Return accounted runtime for the task. ++ * In case the task is currently running, return the runtime plus current's ++ * pending runtime that have not been accounted yet. ++ */ ++unsigned long long task_sched_runtime(struct task_struct *p) ++{ ++ unsigned long flags; ++ struct rq *rq; ++ u64 ns; ++ ++#if defined(CONFIG_64BIT) && defined(CONFIG_SMP) ++ /* ++ * 64-bit doesn't need locks to atomically read a 64bit value. ++ * So we have a optimization chance when the task's delta_exec is 0. ++ * Reading ->on_cpu is racy, but this is ok. ++ * ++ * If we race with it leaving cpu, we'll take a lock. So we're correct. ++ * If we race with it entering cpu, unaccounted time is 0. This is ++ * indistinguishable from the read occurring a few cycles earlier. ++ * If we see ->on_cpu without ->on_rq, the task is leaving, and has ++ * been accounted, so we're correct here as well. ++ */ ++ if (!p->on_cpu || !task_on_rq_queued(p)) ++ return p->se.sum_exec_runtime; ++#endif ++ ++ rq = task_rq_lock(p, &flags); ++ /* ++ * Must be ->curr _and_ ->on_rq. If dequeued, we would ++ * project cycles that may never be accounted to this ++ * thread, breaking clock_gettime(). ++ */ ++ if (task_current(rq, p) && task_on_rq_queued(p)) { ++ update_rq_clock(rq); ++ p->sched_class->update_curr(rq); ++ } ++ ns = p->se.sum_exec_runtime; ++ task_rq_unlock(rq, p, &flags); ++ ++ return ns; ++} ++ ++/* ++ * This function gets called by the timer code, with HZ frequency. ++ * We call it with interrupts disabled. ++ */ ++void scheduler_tick(void) ++{ ++ int cpu = smp_processor_id(); ++ struct rq *rq = cpu_rq(cpu); ++ struct task_struct *curr = rq->curr; ++ ++ sched_clock_tick(); ++ ++ raw_spin_lock(&rq->lock); ++ update_rq_clock(rq); ++ curr->sched_class->task_tick(rq, curr, 0); ++ update_cpu_load_active(rq); ++ raw_spin_unlock(&rq->lock); ++ ++ perf_event_task_tick(); ++ ++#ifdef CONFIG_SMP ++ rq->idle_balance = idle_cpu(cpu); ++ trigger_load_balance(rq); ++#endif ++ rq_last_tick_reset(rq); ++} ++ ++#ifdef CONFIG_NO_HZ_FULL ++/** ++ * scheduler_tick_max_deferment ++ * ++ * Keep at least one tick per second when a single ++ * active task is running because the scheduler doesn't ++ * yet completely support full dynticks environment. ++ * ++ * This makes sure that uptime, CFS vruntime, load ++ * balancing, etc... continue to move forward, even ++ * with a very low granularity. ++ * ++ * Return: Maximum deferment in nanoseconds. ++ */ ++u64 scheduler_tick_max_deferment(void) ++{ ++ struct rq *rq = this_rq(); ++ unsigned long next, now = ACCESS_ONCE(jiffies); ++ ++ next = rq->last_sched_tick + HZ; ++ ++ if (time_before_eq(next, now)) ++ return 0; ++ ++ return jiffies_to_nsecs(next - now); ++} ++#endif ++ ++notrace unsigned long get_parent_ip(unsigned long addr) ++{ ++ if (in_lock_functions(addr)) { ++ addr = CALLER_ADDR2; ++ if (in_lock_functions(addr)) ++ addr = CALLER_ADDR3; ++ } ++ return addr; ++} ++ ++#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ ++ defined(CONFIG_PREEMPT_TRACER)) ++ ++void preempt_count_add(int val) ++{ ++#ifdef CONFIG_DEBUG_PREEMPT ++ /* ++ * Underflow? ++ */ ++ if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) ++ return; ++#endif ++ __preempt_count_add(val); ++#ifdef CONFIG_DEBUG_PREEMPT ++ /* ++ * Spinlock count overflowing soon? ++ */ ++ DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= ++ PREEMPT_MASK - 10); ++#endif ++ if (preempt_count() == val) { ++ unsigned long ip = get_parent_ip(CALLER_ADDR1); ++#ifdef CONFIG_DEBUG_PREEMPT ++ current->preempt_disable_ip = ip; ++#endif ++ trace_preempt_off(CALLER_ADDR0, ip); ++ } ++} ++EXPORT_SYMBOL(preempt_count_add); ++NOKPROBE_SYMBOL(preempt_count_add); ++ ++void preempt_count_sub(int val) ++{ ++#ifdef CONFIG_DEBUG_PREEMPT ++ /* ++ * Underflow? ++ */ ++ if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) ++ return; ++ /* ++ * Is the spinlock portion underflowing? ++ */ ++ if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && ++ !(preempt_count() & PREEMPT_MASK))) ++ return; ++#endif ++ ++ if (preempt_count() == val) ++ trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); ++ __preempt_count_sub(val); ++} ++EXPORT_SYMBOL(preempt_count_sub); ++NOKPROBE_SYMBOL(preempt_count_sub); ++ ++#endif ++ ++/* ++ * Print scheduling while atomic bug: ++ */ ++static noinline void __schedule_bug(struct task_struct *prev) ++{ ++ if (oops_in_progress) ++ return; ++ ++ printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", ++ prev->comm, prev->pid, preempt_count()); ++ ++ debug_show_held_locks(prev); ++ print_modules(); ++ if (irqs_disabled()) ++ print_irqtrace_events(prev); ++#ifdef CONFIG_DEBUG_PREEMPT ++ if (in_atomic_preempt_off()) { ++ pr_err("Preemption disabled at:"); ++ print_ip_sym(current->preempt_disable_ip); ++ pr_cont("\n"); ++ } ++#endif ++ dump_stack(); ++ add_taint(TAINT_WARN, LOCKDEP_STILL_OK); ++} ++ ++/* ++ * Various schedule()-time debugging checks and statistics: ++ */ ++static inline void schedule_debug(struct task_struct *prev) ++{ ++#ifdef CONFIG_SCHED_STACK_END_CHECK ++ BUG_ON(unlikely(task_stack_end_corrupted(prev))); ++#endif ++ /* ++ * Test if we are atomic. Since do_exit() needs to call into ++ * schedule() atomically, we ignore that path. Otherwise whine ++ * if we are scheduling when we should not. ++ */ ++ if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD)) ++ __schedule_bug(prev); ++ rcu_sleep_check(); ++ ++ profile_hit(SCHED_PROFILING, __builtin_return_address(0)); ++ ++ schedstat_inc(this_rq(), sched_count); ++} ++ ++/* ++ * Pick up the highest-prio task: ++ */ ++static inline struct task_struct * ++pick_next_task(struct rq *rq, struct task_struct *prev) ++{ ++ const struct sched_class *class = &fair_sched_class; ++ struct task_struct *p; ++ ++ /* ++ * Optimization: we know that if all tasks are in ++ * the fair class we can call that function directly: ++ */ ++ if (likely(prev->sched_class == class && ++ rq->nr_running == rq->cfs.h_nr_running)) { ++ p = fair_sched_class.pick_next_task(rq, prev); ++ if (unlikely(p == RETRY_TASK)) ++ goto again; ++ ++ /* assumes fair_sched_class->next == idle_sched_class */ ++ if (unlikely(!p)) ++ p = idle_sched_class.pick_next_task(rq, prev); ++ ++ return p; ++ } ++ ++again: ++ for_each_class(class) { ++ p = class->pick_next_task(rq, prev); ++ if (p) { ++ if (unlikely(p == RETRY_TASK)) ++ goto again; ++ return p; ++ } ++ } ++ ++ BUG(); /* the idle class will always have a runnable task */ ++} ++ ++/* ++ * __schedule() is the main scheduler function. ++ * ++ * The main means of driving the scheduler and thus entering this function are: ++ * ++ * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. ++ * ++ * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return ++ * paths. For example, see arch/x86/entry_64.S. ++ * ++ * To drive preemption between tasks, the scheduler sets the flag in timer ++ * interrupt handler scheduler_tick(). ++ * ++ * 3. Wakeups don't really cause entry into schedule(). They add a ++ * task to the run-queue and that's it. ++ * ++ * Now, if the new task added to the run-queue preempts the current ++ * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets ++ * called on the nearest possible occasion: ++ * ++ * - If the kernel is preemptible (CONFIG_PREEMPT=y): ++ * ++ * - in syscall or exception context, at the next outmost ++ * preempt_enable(). (this might be as soon as the wake_up()'s ++ * spin_unlock()!) ++ * ++ * - in IRQ context, return from interrupt-handler to ++ * preemptible context ++ * ++ * - If the kernel is not preemptible (CONFIG_PREEMPT is not set) ++ * then at the next: ++ * ++ * - cond_resched() call ++ * - explicit schedule() call ++ * - return from syscall or exception to user-space ++ * - return from interrupt-handler to user-space ++ * ++ * WARNING: all callers must re-check need_resched() afterward and reschedule ++ * accordingly in case an event triggered the need for rescheduling (such as ++ * an interrupt waking up a task) while preemption was disabled in __schedule(). ++ */ ++static void __sched __schedule(void) ++{ ++ struct task_struct *prev, *next; ++ unsigned long *switch_count; ++ struct rq *rq; ++ int cpu; ++ ++ preempt_disable(); ++ cpu = smp_processor_id(); ++ rq = cpu_rq(cpu); ++ rcu_note_context_switch(); ++ prev = rq->curr; ++ ++ schedule_debug(prev); ++ ++ if (sched_feat(HRTICK)) ++ hrtick_clear(rq); ++ ++ /* ++ * Make sure that signal_pending_state()->signal_pending() below ++ * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) ++ * done by the caller to avoid the race with signal_wake_up(). ++ */ ++ smp_mb__before_spinlock(); ++ raw_spin_lock_irq(&rq->lock); ++ ++ rq->clock_skip_update <<= 1; /* promote REQ to ACT */ ++ ++ switch_count = &prev->nivcsw; ++ if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { ++ if (unlikely(signal_pending_state(prev->state, prev))) { ++ prev->state = TASK_RUNNING; ++ } else { ++ deactivate_task(rq, prev, DEQUEUE_SLEEP); ++ prev->on_rq = 0; ++ ++ /* ++ * If a worker went to sleep, notify and ask workqueue ++ * whether it wants to wake up a task to maintain ++ * concurrency. ++ */ ++ if (prev->flags & PF_WQ_WORKER) { ++ struct task_struct *to_wakeup; ++ ++ to_wakeup = wq_worker_sleeping(prev, cpu); ++ if (to_wakeup) ++ try_to_wake_up_local(to_wakeup); ++ } ++ } ++ switch_count = &prev->nvcsw; ++ } ++ ++ if (task_on_rq_queued(prev)) ++ update_rq_clock(rq); ++ ++ next = pick_next_task(rq, prev); ++ clear_tsk_need_resched(prev); ++ clear_preempt_need_resched(); ++ rq->clock_skip_update = 0; ++ ++ if (likely(prev != next)) { ++ rq->nr_switches++; ++ rq->curr = next; ++ ++*switch_count; ++ ++ rq = context_switch(rq, prev, next); /* unlocks the rq */ ++ cpu = cpu_of(rq); ++ } else ++ raw_spin_unlock_irq(&rq->lock); ++ ++ post_schedule(rq); ++ ++ sched_preempt_enable_no_resched(); ++} ++ ++static inline void sched_submit_work(struct task_struct *tsk) ++{ ++ if (!tsk->state || tsk_is_pi_blocked(tsk)) ++ return; ++ /* ++ * If we are going to sleep and we have plugged IO queued, ++ * make sure to submit it to avoid deadlocks. ++ */ ++ if (blk_needs_flush_plug(tsk)) ++ blk_schedule_flush_plug(tsk); ++} ++ ++asmlinkage __visible void __sched schedule(void) ++{ ++ struct task_struct *tsk = current; ++ ++ sched_submit_work(tsk); ++ do { ++ __schedule(); ++ } while (need_resched()); ++} ++EXPORT_SYMBOL(schedule); ++ ++#ifdef CONFIG_CONTEXT_TRACKING ++asmlinkage __visible void __sched schedule_user(void) ++{ ++ /* ++ * If we come here after a random call to set_need_resched(), ++ * or we have been woken up remotely but the IPI has not yet arrived, ++ * we haven't yet exited the RCU idle mode. Do it here manually until ++ * we find a better solution. ++ * ++ * NB: There are buggy callers of this function. Ideally we ++ * should warn if prev_state != CONTEXT_USER, but that will trigger ++ * too frequently to make sense yet. ++ */ ++ enum ctx_state prev_state = exception_enter(); ++ schedule(); ++ exception_exit(prev_state); ++} ++#endif ++ ++/** ++ * schedule_preempt_disabled - called with preemption disabled ++ * ++ * Returns with preemption disabled. Note: preempt_count must be 1 ++ */ ++void __sched schedule_preempt_disabled(void) ++{ ++ sched_preempt_enable_no_resched(); ++ schedule(); ++ preempt_disable(); ++} ++ ++static void __sched notrace preempt_schedule_common(void) ++{ ++ do { ++ __preempt_count_add(PREEMPT_ACTIVE); ++ __schedule(); ++ __preempt_count_sub(PREEMPT_ACTIVE); ++ ++ /* ++ * Check again in case we missed a preemption opportunity ++ * between schedule and now. ++ */ ++ barrier(); ++ } while (need_resched()); ++} ++ ++#ifdef CONFIG_PREEMPT ++/* ++ * this is the entry point to schedule() from in-kernel preemption ++ * off of preempt_enable. Kernel preemptions off return from interrupt ++ * occur there and call schedule directly. ++ */ ++asmlinkage __visible void __sched notrace preempt_schedule(void) ++{ ++ /* ++ * If there is a non-zero preempt_count or interrupts are disabled, ++ * we do not want to preempt the current task. Just return.. ++ */ ++ if (likely(!preemptible())) ++ return; ++ ++ preempt_schedule_common(); ++} ++NOKPROBE_SYMBOL(preempt_schedule); ++EXPORT_SYMBOL(preempt_schedule); ++ ++#ifdef CONFIG_CONTEXT_TRACKING ++/** ++ * preempt_schedule_context - preempt_schedule called by tracing ++ * ++ * The tracing infrastructure uses preempt_enable_notrace to prevent ++ * recursion and tracing preempt enabling caused by the tracing ++ * infrastructure itself. But as tracing can happen in areas coming ++ * from userspace or just about to enter userspace, a preempt enable ++ * can occur before user_exit() is called. This will cause the scheduler ++ * to be called when the system is still in usermode. ++ * ++ * To prevent this, the preempt_enable_notrace will use this function ++ * instead of preempt_schedule() to exit user context if needed before ++ * calling the scheduler. ++ */ ++asmlinkage __visible void __sched notrace preempt_schedule_context(void) ++{ ++ enum ctx_state prev_ctx; ++ ++ if (likely(!preemptible())) ++ return; ++ ++ do { ++ __preempt_count_add(PREEMPT_ACTIVE); ++ /* ++ * Needs preempt disabled in case user_exit() is traced ++ * and the tracer calls preempt_enable_notrace() causing ++ * an infinite recursion. ++ */ ++ prev_ctx = exception_enter(); ++ __schedule(); ++ exception_exit(prev_ctx); ++ ++ __preempt_count_sub(PREEMPT_ACTIVE); ++ barrier(); ++ } while (need_resched()); ++} ++EXPORT_SYMBOL_GPL(preempt_schedule_context); ++#endif /* CONFIG_CONTEXT_TRACKING */ ++ ++#endif /* CONFIG_PREEMPT */ ++ ++/* ++ * this is the entry point to schedule() from kernel preemption ++ * off of irq context. ++ * Note, that this is called and return with irqs disabled. This will ++ * protect us against recursive calling from irq. ++ */ ++asmlinkage __visible void __sched preempt_schedule_irq(void) ++{ ++ enum ctx_state prev_state; ++ ++ /* Catch callers which need to be fixed */ ++ BUG_ON(preempt_count() || !irqs_disabled()); ++ ++ prev_state = exception_enter(); ++ ++ do { ++ __preempt_count_add(PREEMPT_ACTIVE); ++ local_irq_enable(); ++ __schedule(); ++ local_irq_disable(); ++ __preempt_count_sub(PREEMPT_ACTIVE); ++ ++ /* ++ * Check again in case we missed a preemption opportunity ++ * between schedule and now. ++ */ ++ barrier(); ++ } while (need_resched()); ++ ++ exception_exit(prev_state); ++} ++ ++int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags, ++ void *key) ++{ ++ return try_to_wake_up(curr->private, mode, wake_flags); ++} ++EXPORT_SYMBOL(default_wake_function); ++ ++#ifdef CONFIG_RT_MUTEXES ++ ++/* ++ * rt_mutex_setprio - set the current priority of a task ++ * @p: task ++ * @prio: prio value (kernel-internal form) ++ * ++ * This function changes the 'effective' priority of a task. It does ++ * not touch ->normal_prio like __setscheduler(). ++ * ++ * Used by the rt_mutex code to implement priority inheritance ++ * logic. Call site only calls if the priority of the task changed. ++ */ ++void rt_mutex_setprio(struct task_struct *p, int prio) ++{ ++ int oldprio, queued, running, enqueue_flag = 0; ++ struct rq *rq; ++ const struct sched_class *prev_class; ++ ++ BUG_ON(prio > MAX_PRIO); ++ ++ rq = __task_rq_lock(p); ++ ++ /* ++ * Idle task boosting is a nono in general. There is one ++ * exception, when PREEMPT_RT and NOHZ is active: ++ * ++ * The idle task calls get_next_timer_interrupt() and holds ++ * the timer wheel base->lock on the CPU and another CPU wants ++ * to access the timer (probably to cancel it). We can safely ++ * ignore the boosting request, as the idle CPU runs this code ++ * with interrupts disabled and will complete the lock ++ * protected section without being interrupted. So there is no ++ * real need to boost. ++ */ ++ if (unlikely(p == rq->idle)) { ++ WARN_ON(p != rq->curr); ++ WARN_ON(p->pi_blocked_on); ++ goto out_unlock; ++ } ++ ++ trace_sched_pi_setprio(p, prio); ++ oldprio = p->prio; ++ prev_class = p->sched_class; ++ queued = task_on_rq_queued(p); ++ running = task_current(rq, p); ++ if (queued) ++ dequeue_task(rq, p, 0); ++ if (running) ++ put_prev_task(rq, p); ++ ++ /* ++ * Boosting condition are: ++ * 1. -rt task is running and holds mutex A ++ * --> -dl task blocks on mutex A ++ * ++ * 2. -dl task is running and holds mutex A ++ * --> -dl task blocks on mutex A and could preempt the ++ * running task ++ */ ++ if (dl_prio(prio)) { ++ struct task_struct *pi_task = rt_mutex_get_top_task(p); ++ if (!dl_prio(p->normal_prio) || ++ (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) { ++ p->dl.dl_boosted = 1; ++ p->dl.dl_throttled = 0; ++ enqueue_flag = ENQUEUE_REPLENISH; ++ } else ++ p->dl.dl_boosted = 0; ++ p->sched_class = &dl_sched_class; ++ } else if (rt_prio(prio)) { ++ if (dl_prio(oldprio)) ++ p->dl.dl_boosted = 0; ++ if (oldprio < prio) ++ enqueue_flag = ENQUEUE_HEAD; ++ p->sched_class = &rt_sched_class; ++ } else { ++ if (dl_prio(oldprio)) ++ p->dl.dl_boosted = 0; ++ if (rt_prio(oldprio)) ++ p->rt.timeout = 0; ++ p->sched_class = &fair_sched_class; ++ } ++ ++ p->prio = prio; ++ ++ if (running) ++ p->sched_class->set_curr_task(rq); ++ if (queued) ++ enqueue_task(rq, p, enqueue_flag); ++ ++ check_class_changed(rq, p, prev_class, oldprio); ++out_unlock: ++ __task_rq_unlock(rq); ++} ++#endif ++ ++void set_user_nice(struct task_struct *p, long nice) ++{ ++ int old_prio, delta, queued; ++ unsigned long flags; ++ struct rq *rq; ++ ++ if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) ++ return; ++ /* ++ * We have to be careful, if called from sys_setpriority(), ++ * the task might be in the middle of scheduling on another CPU. ++ */ ++ rq = task_rq_lock(p, &flags); ++ /* ++ * The RT priorities are set via sched_setscheduler(), but we still ++ * allow the 'normal' nice value to be set - but as expected ++ * it wont have any effect on scheduling until the task is ++ * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: ++ */ ++ if (task_has_dl_policy(p) || task_has_rt_policy(p)) { ++ p->static_prio = NICE_TO_PRIO(nice); ++ goto out_unlock; ++ } ++ queued = task_on_rq_queued(p); ++ if (queued) ++ dequeue_task(rq, p, 0); ++ ++ p->static_prio = NICE_TO_PRIO(nice); ++ set_load_weight(p); ++ old_prio = p->prio; ++ p->prio = effective_prio(p); ++ delta = p->prio - old_prio; ++ ++ if (queued) { ++ enqueue_task(rq, p, 0); ++ /* ++ * If the task increased its priority or is running and ++ * lowered its priority, then reschedule its CPU: ++ */ ++ if (delta < 0 || (delta > 0 && task_running(rq, p))) ++ resched_curr(rq); ++ } ++out_unlock: ++ task_rq_unlock(rq, p, &flags); ++} ++EXPORT_SYMBOL(set_user_nice); ++ ++/* ++ * can_nice - check if a task can reduce its nice value ++ * @p: task ++ * @nice: nice value ++ */ ++int can_nice(const struct task_struct *p, const int nice) ++{ ++ /* convert nice value [19,-20] to rlimit style value [1,40] */ ++ int nice_rlim = nice_to_rlimit(nice); ++ ++ return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || ++ capable(CAP_SYS_NICE)); ++} ++ ++#ifdef __ARCH_WANT_SYS_NICE ++ ++/* ++ * sys_nice - change the priority of the current process. ++ * @increment: priority increment ++ * ++ * sys_setpriority is a more generic, but much slower function that ++ * does similar things. ++ */ ++SYSCALL_DEFINE1(nice, int, increment) ++{ ++ long nice, retval; ++ ++ /* ++ * Setpriority might change our priority at the same moment. ++ * We don't have to worry. Conceptually one call occurs first ++ * and we have a single winner. ++ */ ++ increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); ++ nice = task_nice(current) + increment; ++ ++ nice = clamp_val(nice, MIN_NICE, MAX_NICE); ++ if (increment < 0 && !can_nice(current, nice)) ++ return -EPERM; ++ ++ retval = security_task_setnice(current, nice); ++ if (retval) ++ return retval; ++ ++ set_user_nice(current, nice); ++ return 0; ++} ++ ++#endif ++ ++/** ++ * task_prio - return the priority value of a given task. ++ * @p: the task in question. ++ * ++ * Return: The priority value as seen by users in /proc. ++ * RT tasks are offset by -200. Normal tasks are centered ++ * around 0, value goes from -16 to +15. ++ */ ++int task_prio(const struct task_struct *p) ++{ ++ return p->prio - MAX_RT_PRIO; ++} ++ ++/** ++ * idle_cpu - is a given cpu idle currently? ++ * @cpu: the processor in question. ++ * ++ * Return: 1 if the CPU is currently idle. 0 otherwise. ++ */ ++int idle_cpu(int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ ++ if (rq->curr != rq->idle) ++ return 0; ++ ++ if (rq->nr_running) ++ return 0; ++ ++#ifdef CONFIG_SMP ++ if (!llist_empty(&rq->wake_list)) ++ return 0; ++#endif ++ ++ return 1; ++} ++ ++/** ++ * idle_task - return the idle task for a given cpu. ++ * @cpu: the processor in question. ++ * ++ * Return: The idle task for the cpu @cpu. ++ */ ++struct task_struct *idle_task(int cpu) ++{ ++ return cpu_rq(cpu)->idle; ++} ++ ++/** ++ * find_process_by_pid - find a process with a matching PID value. ++ * @pid: the pid in question. ++ * ++ * The task of @pid, if found. %NULL otherwise. ++ */ ++static struct task_struct *find_process_by_pid(pid_t pid) ++{ ++ return pid ? find_task_by_vpid(pid) : current; ++} ++ ++/* ++ * This function initializes the sched_dl_entity of a newly becoming ++ * SCHED_DEADLINE task. ++ * ++ * Only the static values are considered here, the actual runtime and the ++ * absolute deadline will be properly calculated when the task is enqueued ++ * for the first time with its new policy. ++ */ ++static void ++__setparam_dl(struct task_struct *p, const struct sched_attr *attr) ++{ ++ struct sched_dl_entity *dl_se = &p->dl; ++ ++ dl_se->dl_runtime = attr->sched_runtime; ++ dl_se->dl_deadline = attr->sched_deadline; ++ dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; ++ dl_se->flags = attr->sched_flags; ++ dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); ++ ++ /* ++ * Changing the parameters of a task is 'tricky' and we're not doing ++ * the correct thing -- also see task_dead_dl() and switched_from_dl(). ++ * ++ * What we SHOULD do is delay the bandwidth release until the 0-lag ++ * point. This would include retaining the task_struct until that time ++ * and change dl_overflow() to not immediately decrement the current ++ * amount. ++ * ++ * Instead we retain the current runtime/deadline and let the new ++ * parameters take effect after the current reservation period lapses. ++ * This is safe (albeit pessimistic) because the 0-lag point is always ++ * before the current scheduling deadline. ++ * ++ * We can still have temporary overloads because we do not delay the ++ * change in bandwidth until that time; so admission control is ++ * not on the safe side. It does however guarantee tasks will never ++ * consume more than promised. ++ */ ++} ++ ++/* ++ * sched_setparam() passes in -1 for its policy, to let the functions ++ * it calls know not to change it. ++ */ ++#define SETPARAM_POLICY -1 ++ ++static void __setscheduler_params(struct task_struct *p, ++ const struct sched_attr *attr) ++{ ++ int policy = attr->sched_policy; ++ ++ if (policy == SETPARAM_POLICY) ++ policy = p->policy; ++ ++ p->policy = policy; ++ ++ if (dl_policy(policy)) ++ __setparam_dl(p, attr); ++ else if (fair_policy(policy)) ++ p->static_prio = NICE_TO_PRIO(attr->sched_nice); ++ ++ /* ++ * __sched_setscheduler() ensures attr->sched_priority == 0 when ++ * !rt_policy. Always setting this ensures that things like ++ * getparam()/getattr() don't report silly values for !rt tasks. ++ */ ++ p->rt_priority = attr->sched_priority; ++ p->normal_prio = normal_prio(p); ++ set_load_weight(p); ++} ++ ++/* Actually do priority change: must hold pi & rq lock. */ ++static void __setscheduler(struct rq *rq, struct task_struct *p, ++ const struct sched_attr *attr, bool keep_boost) ++{ ++ __setscheduler_params(p, attr); ++ ++ /* ++ * Keep a potential priority boosting if called from ++ * sched_setscheduler(). ++ */ ++ if (keep_boost) ++ p->prio = rt_mutex_get_effective_prio(p, normal_prio(p)); ++ else ++ p->prio = normal_prio(p); ++ ++ if (dl_prio(p->prio)) ++ p->sched_class = &dl_sched_class; ++ else if (rt_prio(p->prio)) ++ p->sched_class = &rt_sched_class; ++ else ++ p->sched_class = &fair_sched_class; ++} ++ ++static void ++__getparam_dl(struct task_struct *p, struct sched_attr *attr) ++{ ++ struct sched_dl_entity *dl_se = &p->dl; ++ ++ attr->sched_priority = p->rt_priority; ++ attr->sched_runtime = dl_se->dl_runtime; ++ attr->sched_deadline = dl_se->dl_deadline; ++ attr->sched_period = dl_se->dl_period; ++ attr->sched_flags = dl_se->flags; ++} ++ ++/* ++ * This function validates the new parameters of a -deadline task. ++ * We ask for the deadline not being zero, and greater or equal ++ * than the runtime, as well as the period of being zero or ++ * greater than deadline. Furthermore, we have to be sure that ++ * user parameters are above the internal resolution of 1us (we ++ * check sched_runtime only since it is always the smaller one) and ++ * below 2^63 ns (we have to check both sched_deadline and ++ * sched_period, as the latter can be zero). ++ */ ++static bool ++__checkparam_dl(const struct sched_attr *attr) ++{ ++ /* deadline != 0 */ ++ if (attr->sched_deadline == 0) ++ return false; ++ ++ /* ++ * Since we truncate DL_SCALE bits, make sure we're at least ++ * that big. ++ */ ++ if (attr->sched_runtime < (1ULL << DL_SCALE)) ++ return false; ++ ++ /* ++ * Since we use the MSB for wrap-around and sign issues, make ++ * sure it's not set (mind that period can be equal to zero). ++ */ ++ if (attr->sched_deadline & (1ULL << 63) || ++ attr->sched_period & (1ULL << 63)) ++ return false; ++ ++ /* runtime <= deadline <= period (if period != 0) */ ++ if ((attr->sched_period != 0 && ++ attr->sched_period < attr->sched_deadline) || ++ attr->sched_deadline < attr->sched_runtime) ++ return false; ++ ++ return true; ++} ++ ++/* ++ * check the target process has a UID that matches the current process's ++ */ ++static bool check_same_owner(struct task_struct *p) ++{ ++ const struct cred *cred = current_cred(), *pcred; ++ bool match; ++ ++ rcu_read_lock(); ++ pcred = __task_cred(p); ++ match = (uid_eq(cred->euid, pcred->euid) || ++ uid_eq(cred->euid, pcred->uid)); ++ rcu_read_unlock(); ++ return match; ++} ++ ++static bool dl_param_changed(struct task_struct *p, ++ const struct sched_attr *attr) ++{ ++ struct sched_dl_entity *dl_se = &p->dl; ++ ++ if (dl_se->dl_runtime != attr->sched_runtime || ++ dl_se->dl_deadline != attr->sched_deadline || ++ dl_se->dl_period != attr->sched_period || ++ dl_se->flags != attr->sched_flags) ++ return true; ++ ++ return false; ++} ++ ++static int __sched_setscheduler(struct task_struct *p, ++ const struct sched_attr *attr, ++ bool user) ++{ ++ int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 : ++ MAX_RT_PRIO - 1 - attr->sched_priority; ++ int retval, oldprio, oldpolicy = -1, queued, running; ++ int new_effective_prio, policy = attr->sched_policy; ++ unsigned long flags; ++ const struct sched_class *prev_class; ++ struct rq *rq; ++ int reset_on_fork; ++ ++ /* may grab non-irq protected spin_locks */ ++ BUG_ON(in_interrupt()); ++recheck: ++ /* double check policy once rq lock held */ ++ if (policy < 0) { ++ reset_on_fork = p->sched_reset_on_fork; ++ policy = oldpolicy = p->policy; ++ } else { ++ reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); ++ ++ if (policy != SCHED_DEADLINE && ++ policy != SCHED_FIFO && policy != SCHED_RR && ++ policy != SCHED_NORMAL && policy != SCHED_BATCH && ++ policy != SCHED_IDLE) ++ return -EINVAL; ++ } ++ ++ if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK)) ++ return -EINVAL; ++ ++ /* ++ * Valid priorities for SCHED_FIFO and SCHED_RR are ++ * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, ++ * SCHED_BATCH and SCHED_IDLE is 0. ++ */ ++ if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || ++ (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) ++ return -EINVAL; ++ if ((dl_policy(policy) && !__checkparam_dl(attr)) || ++ (rt_policy(policy) != (attr->sched_priority != 0))) ++ return -EINVAL; ++ ++ /* ++ * Allow unprivileged RT tasks to decrease priority: ++ */ ++ if (user && !capable(CAP_SYS_NICE)) { ++ if (fair_policy(policy)) { ++ if (attr->sched_nice < task_nice(p) && ++ !can_nice(p, attr->sched_nice)) ++ return -EPERM; ++ } ++ ++ if (rt_policy(policy)) { ++ unsigned long rlim_rtprio = ++ task_rlimit(p, RLIMIT_RTPRIO); ++ ++ /* can't set/change the rt policy */ ++ if (policy != p->policy && !rlim_rtprio) ++ return -EPERM; ++ ++ /* can't increase priority */ ++ if (attr->sched_priority > p->rt_priority && ++ attr->sched_priority > rlim_rtprio) ++ return -EPERM; ++ } ++ ++ /* ++ * Can't set/change SCHED_DEADLINE policy at all for now ++ * (safest behavior); in the future we would like to allow ++ * unprivileged DL tasks to increase their relative deadline ++ * or reduce their runtime (both ways reducing utilization) ++ */ ++ if (dl_policy(policy)) ++ return -EPERM; ++ ++ /* ++ * Treat SCHED_IDLE as nice 20. Only allow a switch to ++ * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. ++ */ ++ if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) { ++ if (!can_nice(p, task_nice(p))) ++ return -EPERM; ++ } ++ ++ /* can't change other user's priorities */ ++ if (!check_same_owner(p)) ++ return -EPERM; ++ ++ /* Normal users shall not reset the sched_reset_on_fork flag */ ++ if (p->sched_reset_on_fork && !reset_on_fork) ++ return -EPERM; ++ } ++ ++ if (user) { ++ retval = security_task_setscheduler(p); ++ if (retval) ++ return retval; ++ } ++ ++ /* ++ * make sure no PI-waiters arrive (or leave) while we are ++ * changing the priority of the task: ++ * ++ * To be able to change p->policy safely, the appropriate ++ * runqueue lock must be held. ++ */ ++ rq = task_rq_lock(p, &flags); ++ ++ /* ++ * Changing the policy of the stop threads its a very bad idea ++ */ ++ if (p == rq->stop) { ++ task_rq_unlock(rq, p, &flags); ++ return -EINVAL; ++ } ++ ++ /* ++ * If not changing anything there's no need to proceed further, ++ * but store a possible modification of reset_on_fork. ++ */ ++ if (unlikely(policy == p->policy)) { ++ if (fair_policy(policy) && attr->sched_nice != task_nice(p)) ++ goto change; ++ if (rt_policy(policy) && attr->sched_priority != p->rt_priority) ++ goto change; ++ if (dl_policy(policy) && dl_param_changed(p, attr)) ++ goto change; ++ ++ p->sched_reset_on_fork = reset_on_fork; ++ task_rq_unlock(rq, p, &flags); ++ return 0; ++ } ++change: ++ ++ if (user) { ++#ifdef CONFIG_RT_GROUP_SCHED ++ /* ++ * Do not allow realtime tasks into groups that have no runtime ++ * assigned. ++ */ ++ if (rt_bandwidth_enabled() && rt_policy(policy) && ++ task_group(p)->rt_bandwidth.rt_runtime == 0 && ++ !task_group_is_autogroup(task_group(p))) { ++ task_rq_unlock(rq, p, &flags); ++ return -EPERM; ++ } ++#endif ++#ifdef CONFIG_SMP ++ if (dl_bandwidth_enabled() && dl_policy(policy)) { ++ cpumask_t *span = rq->rd->span; ++ ++ /* ++ * Don't allow tasks with an affinity mask smaller than ++ * the entire root_domain to become SCHED_DEADLINE. We ++ * will also fail if there's no bandwidth available. ++ */ ++ if (!cpumask_subset(span, &p->cpus_allowed) || ++ rq->rd->dl_bw.bw == 0) { ++ task_rq_unlock(rq, p, &flags); ++ return -EPERM; ++ } ++ } ++#endif ++ } ++ ++ /* recheck policy now with rq lock held */ ++ if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { ++ policy = oldpolicy = -1; ++ task_rq_unlock(rq, p, &flags); ++ goto recheck; ++ } ++ ++ /* ++ * If setscheduling to SCHED_DEADLINE (or changing the parameters ++ * of a SCHED_DEADLINE task) we need to check if enough bandwidth ++ * is available. ++ */ ++ if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) { ++ task_rq_unlock(rq, p, &flags); ++ return -EBUSY; ++ } ++ ++ p->sched_reset_on_fork = reset_on_fork; ++ oldprio = p->prio; ++ ++ /* ++ * Take priority boosted tasks into account. If the new ++ * effective priority is unchanged, we just store the new ++ * normal parameters and do not touch the scheduler class and ++ * the runqueue. This will be done when the task deboost ++ * itself. ++ */ ++ new_effective_prio = rt_mutex_get_effective_prio(p, newprio); ++ if (new_effective_prio == oldprio) { ++ __setscheduler_params(p, attr); ++ task_rq_unlock(rq, p, &flags); ++ return 0; ++ } ++ ++ queued = task_on_rq_queued(p); ++ running = task_current(rq, p); ++ if (queued) ++ dequeue_task(rq, p, 0); ++ if (running) ++ put_prev_task(rq, p); ++ ++ prev_class = p->sched_class; ++ __setscheduler(rq, p, attr, true); ++ ++ if (running) ++ p->sched_class->set_curr_task(rq); ++ if (queued) { ++ /* ++ * We enqueue to tail when the priority of a task is ++ * increased (user space view). ++ */ ++ enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0); ++ } ++ ++ check_class_changed(rq, p, prev_class, oldprio); ++ task_rq_unlock(rq, p, &flags); ++ ++ rt_mutex_adjust_pi(p); ++ ++ return 0; ++} ++ ++static int _sched_setscheduler(struct task_struct *p, int policy, ++ const struct sched_param *param, bool check) ++{ ++ struct sched_attr attr = { ++ .sched_policy = policy, ++ .sched_priority = param->sched_priority, ++ .sched_nice = PRIO_TO_NICE(p->static_prio), ++ }; ++ ++ /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ ++ if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { ++ attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; ++ policy &= ~SCHED_RESET_ON_FORK; ++ attr.sched_policy = policy; ++ } ++ ++ return __sched_setscheduler(p, &attr, check); ++} ++/** ++ * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. ++ * @p: the task in question. ++ * @policy: new policy. ++ * @param: structure containing the new RT priority. ++ * ++ * Return: 0 on success. An error code otherwise. ++ * ++ * NOTE that the task may be already dead. ++ */ ++int sched_setscheduler(struct task_struct *p, int policy, ++ const struct sched_param *param) ++{ ++ return _sched_setscheduler(p, policy, param, true); ++} ++EXPORT_SYMBOL_GPL(sched_setscheduler); ++ ++int sched_setattr(struct task_struct *p, const struct sched_attr *attr) ++{ ++ return __sched_setscheduler(p, attr, true); ++} ++EXPORT_SYMBOL_GPL(sched_setattr); ++ ++/** ++ * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. ++ * @p: the task in question. ++ * @policy: new policy. ++ * @param: structure containing the new RT priority. ++ * ++ * Just like sched_setscheduler, only don't bother checking if the ++ * current context has permission. For example, this is needed in ++ * stop_machine(): we create temporary high priority worker threads, ++ * but our caller might not have that capability. ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++int sched_setscheduler_nocheck(struct task_struct *p, int policy, ++ const struct sched_param *param) ++{ ++ return _sched_setscheduler(p, policy, param, false); ++} ++ ++static int ++do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) ++{ ++ struct sched_param lparam; ++ struct task_struct *p; ++ int retval; ++ ++ if (!param || pid < 0) ++ return -EINVAL; ++ if (copy_from_user(&lparam, param, sizeof(struct sched_param))) ++ return -EFAULT; ++ ++ rcu_read_lock(); ++ retval = -ESRCH; ++ p = find_process_by_pid(pid); ++ if (p != NULL) ++ retval = sched_setscheduler(p, policy, &lparam); ++ rcu_read_unlock(); ++ ++ return retval; ++} ++ ++/* ++ * Mimics kernel/events/core.c perf_copy_attr(). ++ */ ++static int sched_copy_attr(struct sched_attr __user *uattr, ++ struct sched_attr *attr) ++{ ++ u32 size; ++ int ret; ++ ++ if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0)) ++ return -EFAULT; ++ ++ /* ++ * zero the full structure, so that a short copy will be nice. ++ */ ++ memset(attr, 0, sizeof(*attr)); ++ ++ ret = get_user(size, &uattr->size); ++ if (ret) ++ return ret; ++ ++ if (size > PAGE_SIZE) /* silly large */ ++ goto err_size; ++ ++ if (!size) /* abi compat */ ++ size = SCHED_ATTR_SIZE_VER0; ++ ++ if (size < SCHED_ATTR_SIZE_VER0) ++ goto err_size; ++ ++ /* ++ * If we're handed a bigger struct than we know of, ++ * ensure all the unknown bits are 0 - i.e. new ++ * user-space does not rely on any kernel feature ++ * extensions we dont know about yet. ++ */ ++ if (size > sizeof(*attr)) { ++ unsigned char __user *addr; ++ unsigned char __user *end; ++ unsigned char val; ++ ++ addr = (void __user *)uattr + sizeof(*attr); ++ end = (void __user *)uattr + size; ++ ++ for (; addr < end; addr++) { ++ ret = get_user(val, addr); ++ if (ret) ++ return ret; ++ if (val) ++ goto err_size; ++ } ++ size = sizeof(*attr); ++ } ++ ++ ret = copy_from_user(attr, uattr, size); ++ if (ret) ++ return -EFAULT; ++ ++ /* ++ * XXX: do we want to be lenient like existing syscalls; or do we want ++ * to be strict and return an error on out-of-bounds values? ++ */ ++ attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); ++ ++ return 0; ++ ++err_size: ++ put_user(sizeof(*attr), &uattr->size); ++ return -E2BIG; ++} ++ ++/** ++ * sys_sched_setscheduler - set/change the scheduler policy and RT priority ++ * @pid: the pid in question. ++ * @policy: new policy. ++ * @param: structure containing the new RT priority. ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, ++ struct sched_param __user *, param) ++{ ++ /* negative values for policy are not valid */ ++ if (policy < 0) ++ return -EINVAL; ++ ++ return do_sched_setscheduler(pid, policy, param); ++} ++ ++/** ++ * sys_sched_setparam - set/change the RT priority of a thread ++ * @pid: the pid in question. ++ * @param: structure containing the new RT priority. ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) ++{ ++ return do_sched_setscheduler(pid, SETPARAM_POLICY, param); ++} ++ ++/** ++ * sys_sched_setattr - same as above, but with extended sched_attr ++ * @pid: the pid in question. ++ * @uattr: structure containing the extended parameters. ++ * @flags: for future extension. ++ */ ++SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, ++ unsigned int, flags) ++{ ++ struct sched_attr attr; ++ struct task_struct *p; ++ int retval; ++ ++ if (!uattr || pid < 0 || flags) ++ return -EINVAL; ++ ++ retval = sched_copy_attr(uattr, &attr); ++ if (retval) ++ return retval; ++ ++ if ((int)attr.sched_policy < 0) ++ return -EINVAL; ++ ++ rcu_read_lock(); ++ retval = -ESRCH; ++ p = find_process_by_pid(pid); ++ if (p != NULL) ++ retval = sched_setattr(p, &attr); ++ rcu_read_unlock(); ++ ++ return retval; ++} ++ ++/** ++ * sys_sched_getscheduler - get the policy (scheduling class) of a thread ++ * @pid: the pid in question. ++ * ++ * Return: On success, the policy of the thread. Otherwise, a negative error ++ * code. ++ */ ++SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) ++{ ++ struct task_struct *p; ++ int retval; ++ ++ if (pid < 0) ++ return -EINVAL; ++ ++ retval = -ESRCH; ++ rcu_read_lock(); ++ p = find_process_by_pid(pid); ++ if (p) { ++ retval = security_task_getscheduler(p); ++ if (!retval) ++ retval = p->policy ++ | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); ++ } ++ rcu_read_unlock(); ++ return retval; ++} ++ ++/** ++ * sys_sched_getparam - get the RT priority of a thread ++ * @pid: the pid in question. ++ * @param: structure containing the RT priority. ++ * ++ * Return: On success, 0 and the RT priority is in @param. Otherwise, an error ++ * code. ++ */ ++SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) ++{ ++ struct sched_param lp = { .sched_priority = 0 }; ++ struct task_struct *p; ++ int retval; ++ ++ if (!param || pid < 0) ++ return -EINVAL; ++ ++ rcu_read_lock(); ++ p = find_process_by_pid(pid); ++ retval = -ESRCH; ++ if (!p) ++ goto out_unlock; ++ ++ retval = security_task_getscheduler(p); ++ if (retval) ++ goto out_unlock; ++ ++ if (task_has_rt_policy(p)) ++ lp.sched_priority = p->rt_priority; ++ rcu_read_unlock(); ++ ++ /* ++ * This one might sleep, we cannot do it with a spinlock held ... ++ */ ++ retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; ++ ++ return retval; ++ ++out_unlock: ++ rcu_read_unlock(); ++ return retval; ++} ++ ++static int sched_read_attr(struct sched_attr __user *uattr, ++ struct sched_attr *attr, ++ unsigned int usize) ++{ ++ int ret; ++ ++ if (!access_ok(VERIFY_WRITE, uattr, usize)) ++ return -EFAULT; ++ ++ /* ++ * If we're handed a smaller struct than we know of, ++ * ensure all the unknown bits are 0 - i.e. old ++ * user-space does not get uncomplete information. ++ */ ++ if (usize < sizeof(*attr)) { ++ unsigned char *addr; ++ unsigned char *end; ++ ++ addr = (void *)attr + usize; ++ end = (void *)attr + sizeof(*attr); ++ ++ for (; addr < end; addr++) { ++ if (*addr) ++ return -EFBIG; ++ } ++ ++ attr->size = usize; ++ } ++ ++ ret = copy_to_user(uattr, attr, attr->size); ++ if (ret) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++/** ++ * sys_sched_getattr - similar to sched_getparam, but with sched_attr ++ * @pid: the pid in question. ++ * @uattr: structure containing the extended parameters. ++ * @size: sizeof(attr) for fwd/bwd comp. ++ * @flags: for future extension. ++ */ ++SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, ++ unsigned int, size, unsigned int, flags) ++{ ++ struct sched_attr attr = { ++ .size = sizeof(struct sched_attr), ++ }; ++ struct task_struct *p; ++ int retval; ++ ++ if (!uattr || pid < 0 || size > PAGE_SIZE || ++ size < SCHED_ATTR_SIZE_VER0 || flags) ++ return -EINVAL; ++ ++ rcu_read_lock(); ++ p = find_process_by_pid(pid); ++ retval = -ESRCH; ++ if (!p) ++ goto out_unlock; ++ ++ retval = security_task_getscheduler(p); ++ if (retval) ++ goto out_unlock; ++ ++ attr.sched_policy = p->policy; ++ if (p->sched_reset_on_fork) ++ attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; ++ if (task_has_dl_policy(p)) ++ __getparam_dl(p, &attr); ++ else if (task_has_rt_policy(p)) ++ attr.sched_priority = p->rt_priority; ++ else ++ attr.sched_nice = task_nice(p); ++ ++ rcu_read_unlock(); ++ ++ retval = sched_read_attr(uattr, &attr, size); ++ return retval; ++ ++out_unlock: ++ rcu_read_unlock(); ++ return retval; ++} ++ ++long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) ++{ ++ cpumask_var_t cpus_allowed, new_mask; ++ struct task_struct *p; ++ int retval; ++ ++ rcu_read_lock(); ++ ++ p = find_process_by_pid(pid); ++ if (!p) { ++ rcu_read_unlock(); ++ return -ESRCH; ++ } ++ ++ /* Prevent p going away */ ++ get_task_struct(p); ++ rcu_read_unlock(); ++ ++ if (p->flags & PF_NO_SETAFFINITY) { ++ retval = -EINVAL; ++ goto out_put_task; ++ } ++ if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { ++ retval = -ENOMEM; ++ goto out_put_task; ++ } ++ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { ++ retval = -ENOMEM; ++ goto out_free_cpus_allowed; ++ } ++ retval = -EPERM; ++ if (!check_same_owner(p)) { ++ rcu_read_lock(); ++ if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { ++ rcu_read_unlock(); ++ goto out_free_new_mask; ++ } ++ rcu_read_unlock(); ++ } ++ ++ retval = security_task_setscheduler(p); ++ if (retval) ++ goto out_free_new_mask; ++ ++ ++ cpuset_cpus_allowed(p, cpus_allowed); ++ cpumask_and(new_mask, in_mask, cpus_allowed); ++ ++ /* ++ * Since bandwidth control happens on root_domain basis, ++ * if admission test is enabled, we only admit -deadline ++ * tasks allowed to run on all the CPUs in the task's ++ * root_domain. ++ */ ++#ifdef CONFIG_SMP ++ if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { ++ rcu_read_lock(); ++ if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) { ++ retval = -EBUSY; ++ rcu_read_unlock(); ++ goto out_free_new_mask; ++ } ++ rcu_read_unlock(); ++ } ++#endif ++again: ++ retval = set_cpus_allowed_ptr(p, new_mask); ++ ++ if (!retval) { ++ cpuset_cpus_allowed(p, cpus_allowed); ++ if (!cpumask_subset(new_mask, cpus_allowed)) { ++ /* ++ * We must have raced with a concurrent cpuset ++ * update. Just reset the cpus_allowed to the ++ * cpuset's cpus_allowed ++ */ ++ cpumask_copy(new_mask, cpus_allowed); ++ goto again; ++ } ++ } ++out_free_new_mask: ++ free_cpumask_var(new_mask); ++out_free_cpus_allowed: ++ free_cpumask_var(cpus_allowed); ++out_put_task: ++ put_task_struct(p); ++ return retval; ++} ++ ++static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, ++ struct cpumask *new_mask) ++{ ++ if (len < cpumask_size()) ++ cpumask_clear(new_mask); ++ else if (len > cpumask_size()) ++ len = cpumask_size(); ++ ++ return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; ++} ++ ++/** ++ * sys_sched_setaffinity - set the cpu affinity of a process ++ * @pid: pid of the process ++ * @len: length in bytes of the bitmask pointed to by user_mask_ptr ++ * @user_mask_ptr: user-space pointer to the new cpu mask ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, ++ unsigned long __user *, user_mask_ptr) ++{ ++ cpumask_var_t new_mask; ++ int retval; ++ ++ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) ++ return -ENOMEM; ++ ++ retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); ++ if (retval == 0) ++ retval = sched_setaffinity(pid, new_mask); ++ free_cpumask_var(new_mask); ++ return retval; ++} ++ ++long sched_getaffinity(pid_t pid, struct cpumask *mask) ++{ ++ struct task_struct *p; ++ unsigned long flags; ++ int retval; ++ ++ rcu_read_lock(); ++ ++ retval = -ESRCH; ++ p = find_process_by_pid(pid); ++ if (!p) ++ goto out_unlock; ++ ++ retval = security_task_getscheduler(p); ++ if (retval) ++ goto out_unlock; ++ ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++ cpumask_and(mask, &p->cpus_allowed, cpu_active_mask); ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++ ++out_unlock: ++ rcu_read_unlock(); ++ ++ return retval; ++} ++ ++/** ++ * sys_sched_getaffinity - get the cpu affinity of a process ++ * @pid: pid of the process ++ * @len: length in bytes of the bitmask pointed to by user_mask_ptr ++ * @user_mask_ptr: user-space pointer to hold the current cpu mask ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, ++ unsigned long __user *, user_mask_ptr) ++{ ++ int ret; ++ cpumask_var_t mask; ++ ++ if ((len * BITS_PER_BYTE) < nr_cpu_ids) ++ return -EINVAL; ++ if (len & (sizeof(unsigned long)-1)) ++ return -EINVAL; ++ ++ if (!alloc_cpumask_var(&mask, GFP_KERNEL)) ++ return -ENOMEM; ++ ++ ret = sched_getaffinity(pid, mask); ++ if (ret == 0) { ++ size_t retlen = min_t(size_t, len, cpumask_size()); ++ ++ if (copy_to_user(user_mask_ptr, mask, retlen)) ++ ret = -EFAULT; ++ else ++ ret = retlen; ++ } ++ free_cpumask_var(mask); ++ ++ return ret; ++} ++ ++/** ++ * sys_sched_yield - yield the current processor to other threads. ++ * ++ * This function yields the current CPU to other tasks. If there are no ++ * other threads running on this CPU then this function will return. ++ * ++ * Return: 0. ++ */ ++SYSCALL_DEFINE0(sched_yield) ++{ ++ struct rq *rq = this_rq_lock(); ++ ++ schedstat_inc(rq, yld_count); ++ current->sched_class->yield_task(rq); ++ ++ /* ++ * Since we are going to call schedule() anyway, there's ++ * no need to preempt or enable interrupts: ++ */ ++ __release(rq->lock); ++ spin_release(&rq->lock.dep_map, 1, _THIS_IP_); ++ do_raw_spin_unlock(&rq->lock); ++ sched_preempt_enable_no_resched(); ++ ++ schedule(); ++ ++ return 0; ++} ++ ++int __sched _cond_resched(void) ++{ ++ if (should_resched()) { ++ preempt_schedule_common(); ++ return 1; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(_cond_resched); ++ ++/* ++ * __cond_resched_lock() - if a reschedule is pending, drop the given lock, ++ * call schedule, and on return reacquire the lock. ++ * ++ * This works OK both with and without CONFIG_PREEMPT. We do strange low-level ++ * operations here to prevent schedule() from being called twice (once via ++ * spin_unlock(), once by hand). ++ */ ++int __cond_resched_lock(spinlock_t *lock) ++{ ++ int resched = should_resched(); ++ int ret = 0; ++ ++ lockdep_assert_held(lock); ++ ++ if (spin_needbreak(lock) || resched) { ++ spin_unlock(lock); ++ if (resched) ++ preempt_schedule_common(); ++ else ++ cpu_relax(); ++ ret = 1; ++ spin_lock(lock); ++ } ++ return ret; ++} ++EXPORT_SYMBOL(__cond_resched_lock); ++ ++int __sched __cond_resched_softirq(void) ++{ ++ BUG_ON(!in_softirq()); ++ ++ if (should_resched()) { ++ local_bh_enable(); ++ preempt_schedule_common(); ++ local_bh_disable(); ++ return 1; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(__cond_resched_softirq); ++ ++/** ++ * yield - yield the current processor to other threads. ++ * ++ * Do not ever use this function, there's a 99% chance you're doing it wrong. ++ * ++ * The scheduler is at all times free to pick the calling task as the most ++ * eligible task to run, if removing the yield() call from your code breaks ++ * it, its already broken. ++ * ++ * Typical broken usage is: ++ * ++ * while (!event) ++ * yield(); ++ * ++ * where one assumes that yield() will let 'the other' process run that will ++ * make event true. If the current task is a SCHED_FIFO task that will never ++ * happen. Never use yield() as a progress guarantee!! ++ * ++ * If you want to use yield() to wait for something, use wait_event(). ++ * If you want to use yield() to be 'nice' for others, use cond_resched(). ++ * If you still want to use yield(), do not! ++ */ ++void __sched yield(void) ++{ ++ set_current_state(TASK_RUNNING); ++ sys_sched_yield(); ++} ++EXPORT_SYMBOL(yield); ++ ++/** ++ * yield_to - yield the current processor to another thread in ++ * your thread group, or accelerate that thread toward the ++ * processor it's on. ++ * @p: target task ++ * @preempt: whether task preemption is allowed or not ++ * ++ * It's the caller's job to ensure that the target task struct ++ * can't go away on us before we can do any checks. ++ * ++ * Return: ++ * true (>0) if we indeed boosted the target task. ++ * false (0) if we failed to boost the target. ++ * -ESRCH if there's no task to yield to. ++ */ ++int __sched yield_to(struct task_struct *p, bool preempt) ++{ ++ struct task_struct *curr = current; ++ struct rq *rq, *p_rq; ++ unsigned long flags; ++ int yielded = 0; ++ ++ local_irq_save(flags); ++ rq = this_rq(); ++ ++again: ++ p_rq = task_rq(p); ++ /* ++ * If we're the only runnable task on the rq and target rq also ++ * has only one task, there's absolutely no point in yielding. ++ */ ++ if (rq->nr_running == 1 && p_rq->nr_running == 1) { ++ yielded = -ESRCH; ++ goto out_irq; ++ } ++ ++ double_rq_lock(rq, p_rq); ++ if (task_rq(p) != p_rq) { ++ double_rq_unlock(rq, p_rq); ++ goto again; ++ } ++ ++ if (!curr->sched_class->yield_to_task) ++ goto out_unlock; ++ ++ if (curr->sched_class != p->sched_class) ++ goto out_unlock; ++ ++ if (task_running(p_rq, p) || p->state) ++ goto out_unlock; ++ ++ yielded = curr->sched_class->yield_to_task(rq, p, preempt); ++ if (yielded) { ++ schedstat_inc(rq, yld_count); ++ /* ++ * Make p's CPU reschedule; pick_next_entity takes care of ++ * fairness. ++ */ ++ if (preempt && rq != p_rq) ++ resched_curr(p_rq); ++ } ++ ++out_unlock: ++ double_rq_unlock(rq, p_rq); ++out_irq: ++ local_irq_restore(flags); ++ ++ if (yielded > 0) ++ schedule(); ++ ++ return yielded; ++} ++EXPORT_SYMBOL_GPL(yield_to); ++ ++/* ++ * This task is about to go to sleep on IO. Increment rq->nr_iowait so ++ * that process accounting knows that this is a task in IO wait state. ++ */ ++long __sched io_schedule_timeout(long timeout) ++{ ++ int old_iowait = current->in_iowait; ++ struct rq *rq; ++ long ret; ++ ++ current->in_iowait = 1; ++ blk_schedule_flush_plug(current); ++ ++ delayacct_blkio_start(); ++ rq = raw_rq(); ++ atomic_inc(&rq->nr_iowait); ++ ret = schedule_timeout(timeout); ++ current->in_iowait = old_iowait; ++ atomic_dec(&rq->nr_iowait); ++ delayacct_blkio_end(); ++ ++ return ret; ++} ++EXPORT_SYMBOL(io_schedule_timeout); ++ ++/** ++ * sys_sched_get_priority_max - return maximum RT priority. ++ * @policy: scheduling class. ++ * ++ * Return: On success, this syscall returns the maximum ++ * rt_priority that can be used by a given scheduling class. ++ * On failure, a negative error code is returned. ++ */ ++SYSCALL_DEFINE1(sched_get_priority_max, int, policy) ++{ ++ int ret = -EINVAL; ++ ++ switch (policy) { ++ case SCHED_FIFO: ++ case SCHED_RR: ++ ret = MAX_USER_RT_PRIO-1; ++ break; ++ case SCHED_DEADLINE: ++ case SCHED_NORMAL: ++ case SCHED_BATCH: ++ case SCHED_IDLE: ++ ret = 0; ++ break; ++ } ++ return ret; ++} ++ ++/** ++ * sys_sched_get_priority_min - return minimum RT priority. ++ * @policy: scheduling class. ++ * ++ * Return: On success, this syscall returns the minimum ++ * rt_priority that can be used by a given scheduling class. ++ * On failure, a negative error code is returned. ++ */ ++SYSCALL_DEFINE1(sched_get_priority_min, int, policy) ++{ ++ int ret = -EINVAL; ++ ++ switch (policy) { ++ case SCHED_FIFO: ++ case SCHED_RR: ++ ret = 1; ++ break; ++ case SCHED_DEADLINE: ++ case SCHED_NORMAL: ++ case SCHED_BATCH: ++ case SCHED_IDLE: ++ ret = 0; ++ } ++ return ret; ++} ++ ++/** ++ * sys_sched_rr_get_interval - return the default timeslice of a process. ++ * @pid: pid of the process. ++ * @interval: userspace pointer to the timeslice value. ++ * ++ * this syscall writes the default timeslice value of a given process ++ * into the user-space timespec buffer. A value of '0' means infinity. ++ * ++ * Return: On success, 0 and the timeslice is in @interval. Otherwise, ++ * an error code. ++ */ ++SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, ++ struct timespec __user *, interval) ++{ ++ struct task_struct *p; ++ unsigned int time_slice; ++ unsigned long flags; ++ struct rq *rq; ++ int retval; ++ struct timespec t; ++ ++ if (pid < 0) ++ return -EINVAL; ++ ++ retval = -ESRCH; ++ rcu_read_lock(); ++ p = find_process_by_pid(pid); ++ if (!p) ++ goto out_unlock; ++ ++ retval = security_task_getscheduler(p); ++ if (retval) ++ goto out_unlock; ++ ++ rq = task_rq_lock(p, &flags); ++ time_slice = 0; ++ if (p->sched_class->get_rr_interval) ++ time_slice = p->sched_class->get_rr_interval(rq, p); ++ task_rq_unlock(rq, p, &flags); ++ ++ rcu_read_unlock(); ++ jiffies_to_timespec(time_slice, &t); ++ retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0; ++ return retval; ++ ++out_unlock: ++ rcu_read_unlock(); ++ return retval; ++} ++ ++static const char stat_nam[] = TASK_STATE_TO_CHAR_STR; ++ ++void sched_show_task(struct task_struct *p) ++{ ++ unsigned long free = 0; ++ int ppid; ++ unsigned long state = p->state; ++ ++ if (state) ++ state = __ffs(state) + 1; ++ printk(KERN_INFO "%-15.15s %c", p->comm, ++ state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?'); ++#if BITS_PER_LONG == 32 ++ if (state == TASK_RUNNING) ++ printk(KERN_CONT " running "); ++ else ++ printk(KERN_CONT " %08lx ", thread_saved_pc(p)); ++#else ++ if (state == TASK_RUNNING) ++ printk(KERN_CONT " running task "); ++ else ++ printk(KERN_CONT " %016lx ", thread_saved_pc(p)); ++#endif ++#ifdef CONFIG_DEBUG_STACK_USAGE ++ free = stack_not_used(p); ++#endif ++ ppid = 0; ++ rcu_read_lock(); ++ if (pid_alive(p)) ++ ppid = task_pid_nr(rcu_dereference(p->real_parent)); ++ rcu_read_unlock(); ++ printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free, ++ task_pid_nr(p), ppid, ++ (unsigned long)task_thread_info(p)->flags); ++ ++ print_worker_info(KERN_INFO, p); ++ show_stack(p, NULL); ++} ++ ++void show_state_filter(unsigned long state_filter) ++{ ++ struct task_struct *g, *p; ++ ++#if BITS_PER_LONG == 32 ++ printk(KERN_INFO ++ " task PC stack pid father\n"); ++#else ++ printk(KERN_INFO ++ " task PC stack pid father\n"); ++#endif ++ rcu_read_lock(); ++ for_each_process_thread(g, p) { ++ /* ++ * reset the NMI-timeout, listing all files on a slow ++ * console might take a lot of time: ++ */ ++ touch_nmi_watchdog(); ++ if (!state_filter || (p->state & state_filter)) ++ sched_show_task(p); ++ } ++ ++ touch_all_softlockup_watchdogs(); ++ ++#ifdef CONFIG_SCHED_DEBUG ++ sysrq_sched_debug_show(); ++#endif ++ rcu_read_unlock(); ++ /* ++ * Only show locks if all tasks are dumped: ++ */ ++ if (!state_filter) ++ debug_show_all_locks(); ++} ++ ++void init_idle_bootup_task(struct task_struct *idle) ++{ ++ idle->sched_class = &idle_sched_class; ++} ++ ++/** ++ * init_idle - set up an idle thread for a given CPU ++ * @idle: task in question ++ * @cpu: cpu the idle task belongs to ++ * ++ * NOTE: this function does not set the idle thread's NEED_RESCHED ++ * flag, to make booting more robust. ++ */ ++void init_idle(struct task_struct *idle, int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ ++ __sched_fork(0, idle); ++ idle->state = TASK_RUNNING; ++ idle->se.exec_start = sched_clock(); ++ ++ do_set_cpus_allowed(idle, cpumask_of(cpu)); ++ /* ++ * We're having a chicken and egg problem, even though we are ++ * holding rq->lock, the cpu isn't yet set to this cpu so the ++ * lockdep check in task_group() will fail. ++ * ++ * Similar case to sched_fork(). / Alternatively we could ++ * use task_rq_lock() here and obtain the other rq->lock. ++ * ++ * Silence PROVE_RCU ++ */ ++ rcu_read_lock(); ++ __set_task_cpu(idle, cpu); ++ rcu_read_unlock(); ++ ++ rq->curr = rq->idle = idle; ++ idle->on_rq = TASK_ON_RQ_QUEUED; ++#if defined(CONFIG_SMP) ++ idle->on_cpu = 1; ++#endif ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ ++ /* Set the preempt count _outside_ the spinlocks! */ ++ init_idle_preempt_count(idle, cpu); ++ ++ /* ++ * The idle tasks have their own, simple scheduling class: ++ */ ++ idle->sched_class = &idle_sched_class; ++ ftrace_graph_init_idle_task(idle, cpu); ++ vtime_init_idle(idle, cpu); ++#if defined(CONFIG_SMP) ++ sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); ++#endif ++} ++ ++int cpuset_cpumask_can_shrink(const struct cpumask *cur, ++ const struct cpumask *trial) ++{ ++ int ret = 1, trial_cpus; ++ struct dl_bw *cur_dl_b; ++ unsigned long flags; ++ ++ if (!cpumask_weight(cur)) ++ return ret; ++ ++ rcu_read_lock_sched(); ++ cur_dl_b = dl_bw_of(cpumask_any(cur)); ++ trial_cpus = cpumask_weight(trial); ++ ++ raw_spin_lock_irqsave(&cur_dl_b->lock, flags); ++ if (cur_dl_b->bw != -1 && ++ cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw) ++ ret = 0; ++ raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags); ++ rcu_read_unlock_sched(); ++ ++ return ret; ++} ++ ++int task_can_attach(struct task_struct *p, ++ const struct cpumask *cs_cpus_allowed) ++{ ++ int ret = 0; ++ ++ /* ++ * Kthreads which disallow setaffinity shouldn't be moved ++ * to a new cpuset; we don't want to change their cpu ++ * affinity and isolating such threads by their set of ++ * allowed nodes is unnecessary. Thus, cpusets are not ++ * applicable for such threads. This prevents checking for ++ * success of set_cpus_allowed_ptr() on all attached tasks ++ * before cpus_allowed may be changed. ++ */ ++ if (p->flags & PF_NO_SETAFFINITY) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++#ifdef CONFIG_SMP ++ if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span, ++ cs_cpus_allowed)) { ++ unsigned int dest_cpu = cpumask_any_and(cpu_active_mask, ++ cs_cpus_allowed); ++ struct dl_bw *dl_b; ++ bool overflow; ++ int cpus; ++ unsigned long flags; ++ ++ rcu_read_lock_sched(); ++ dl_b = dl_bw_of(dest_cpu); ++ raw_spin_lock_irqsave(&dl_b->lock, flags); ++ cpus = dl_bw_cpus(dest_cpu); ++ overflow = __dl_overflow(dl_b, cpus, 0, p->dl.dl_bw); ++ if (overflow) ++ ret = -EBUSY; ++ else { ++ /* ++ * We reserve space for this task in the destination ++ * root_domain, as we can't fail after this point. ++ * We will free resources in the source root_domain ++ * later on (see set_cpus_allowed_dl()). ++ */ ++ __dl_add(dl_b, p->dl.dl_bw); ++ } ++ raw_spin_unlock_irqrestore(&dl_b->lock, flags); ++ rcu_read_unlock_sched(); ++ ++ } ++#endif ++out: ++ return ret; ++} ++ ++#ifdef CONFIG_SMP ++/* ++ * move_queued_task - move a queued task to new rq. ++ * ++ * Returns (locked) new rq. Old rq's lock is released. ++ */ ++static struct rq *move_queued_task(struct task_struct *p, int new_cpu) ++{ ++ struct rq *rq = task_rq(p); ++ ++ lockdep_assert_held(&rq->lock); ++ ++ dequeue_task(rq, p, 0); ++ p->on_rq = TASK_ON_RQ_MIGRATING; ++ set_task_cpu(p, new_cpu); ++ raw_spin_unlock(&rq->lock); ++ ++ rq = cpu_rq(new_cpu); ++ ++ raw_spin_lock(&rq->lock); ++ BUG_ON(task_cpu(p) != new_cpu); ++ p->on_rq = TASK_ON_RQ_QUEUED; ++ enqueue_task(rq, p, 0); ++ check_preempt_curr(rq, p, 0); ++ ++ return rq; ++} ++ ++void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) ++{ ++ if (p->sched_class->set_cpus_allowed) ++ p->sched_class->set_cpus_allowed(p, new_mask); ++ ++ cpumask_copy(&p->cpus_allowed, new_mask); ++ p->nr_cpus_allowed = cpumask_weight(new_mask); ++} ++ ++/* ++ * This is how migration works: ++ * ++ * 1) we invoke migration_cpu_stop() on the target CPU using ++ * stop_one_cpu(). ++ * 2) stopper starts to run (implicitly forcing the migrated thread ++ * off the CPU) ++ * 3) it checks whether the migrated task is still in the wrong runqueue. ++ * 4) if it's in the wrong runqueue then the migration thread removes ++ * it and puts it into the right queue. ++ * 5) stopper completes and stop_one_cpu() returns and the migration ++ * is done. ++ */ ++ ++/* ++ * Change a given task's CPU affinity. Migrate the thread to a ++ * proper CPU and schedule it away if the CPU it's executing on ++ * is removed from the allowed bitmask. ++ * ++ * NOTE: the caller must have a valid reference to the task, the ++ * task must not exit() & deallocate itself prematurely. The ++ * call is not atomic; no spinlocks may be held. ++ */ ++int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) ++{ ++ unsigned long flags; ++ struct rq *rq; ++ unsigned int dest_cpu; ++ int ret = 0; ++ ++ rq = task_rq_lock(p, &flags); ++ ++ if (cpumask_equal(&p->cpus_allowed, new_mask)) ++ goto out; ++ ++ if (!cpumask_intersects(new_mask, cpu_active_mask)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ do_set_cpus_allowed(p, new_mask); ++ ++ /* Can the task run on the task's current CPU? If so, we're done */ ++ if (cpumask_test_cpu(task_cpu(p), new_mask)) ++ goto out; ++ ++ dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); ++ if (task_running(rq, p) || p->state == TASK_WAKING) { ++ struct migration_arg arg = { p, dest_cpu }; ++ /* Need help from migration thread: drop lock and wait. */ ++ task_rq_unlock(rq, p, &flags); ++ stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); ++ tlb_migrate_finish(p->mm); ++ return 0; ++ } else if (task_on_rq_queued(p)) ++ rq = move_queued_task(p, dest_cpu); ++out: ++ task_rq_unlock(rq, p, &flags); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); ++ ++/* ++ * Move (not current) task off this cpu, onto dest cpu. We're doing ++ * this because either it can't run here any more (set_cpus_allowed() ++ * away from this CPU, or CPU going down), or because we're ++ * attempting to rebalance this task on exec (sched_exec). ++ * ++ * So we race with normal scheduler movements, but that's OK, as long ++ * as the task is no longer on this CPU. ++ * ++ * Returns non-zero if task was successfully migrated. ++ */ ++static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) ++{ ++ struct rq *rq; ++ int ret = 0; ++ ++ if (unlikely(!cpu_active(dest_cpu))) ++ return ret; ++ ++ rq = cpu_rq(src_cpu); ++ ++ raw_spin_lock(&p->pi_lock); ++ raw_spin_lock(&rq->lock); ++ /* Already moved. */ ++ if (task_cpu(p) != src_cpu) ++ goto done; ++ ++ /* Affinity changed (again). */ ++ if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) ++ goto fail; ++ ++ /* ++ * If we're not on a rq, the next wake-up will ensure we're ++ * placed properly. ++ */ ++ if (task_on_rq_queued(p)) ++ rq = move_queued_task(p, dest_cpu); ++done: ++ ret = 1; ++fail: ++ raw_spin_unlock(&rq->lock); ++ raw_spin_unlock(&p->pi_lock); ++ return ret; ++} ++ ++#ifdef CONFIG_NUMA_BALANCING ++/* Migrate current task p to target_cpu */ ++int migrate_task_to(struct task_struct *p, int target_cpu) ++{ ++ struct migration_arg arg = { p, target_cpu }; ++ int curr_cpu = task_cpu(p); ++ ++ if (curr_cpu == target_cpu) ++ return 0; ++ ++ if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p))) ++ return -EINVAL; ++ ++ /* TODO: This is not properly updating schedstats */ ++ ++ trace_sched_move_numa(p, curr_cpu, target_cpu); ++ return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); ++} ++ ++/* ++ * Requeue a task on a given node and accurately track the number of NUMA ++ * tasks on the runqueues ++ */ ++void sched_setnuma(struct task_struct *p, int nid) ++{ ++ struct rq *rq; ++ unsigned long flags; ++ bool queued, running; ++ ++ rq = task_rq_lock(p, &flags); ++ queued = task_on_rq_queued(p); ++ running = task_current(rq, p); ++ ++ if (queued) ++ dequeue_task(rq, p, 0); ++ if (running) ++ put_prev_task(rq, p); ++ ++ p->numa_preferred_nid = nid; ++ ++ if (running) ++ p->sched_class->set_curr_task(rq); ++ if (queued) ++ enqueue_task(rq, p, 0); ++ task_rq_unlock(rq, p, &flags); ++} ++#endif ++ ++/* ++ * migration_cpu_stop - this will be executed by a highprio stopper thread ++ * and performs thread migration by bumping thread off CPU then ++ * 'pushing' onto another runqueue. ++ */ ++static int migration_cpu_stop(void *data) ++{ ++ struct migration_arg *arg = data; ++ ++ /* ++ * The original target cpu might have gone down and we might ++ * be on another cpu but it doesn't matter. ++ */ ++ local_irq_disable(); ++ /* ++ * We need to explicitly wake pending tasks before running ++ * __migrate_task() such that we will not miss enforcing cpus_allowed ++ * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. ++ */ ++ sched_ttwu_pending(); ++ __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu); ++ local_irq_enable(); ++ return 0; ++} ++ ++#ifdef CONFIG_HOTPLUG_CPU ++ ++/* ++ * Ensures that the idle task is using init_mm right before its cpu goes ++ * offline. ++ */ ++void idle_task_exit(void) ++{ ++ struct mm_struct *mm = current->active_mm; ++ ++ BUG_ON(cpu_online(smp_processor_id())); ++ ++ if (mm != &init_mm) { ++ switch_mm(mm, &init_mm, current); ++ finish_arch_post_lock_switch(); ++ } ++ mmdrop(mm); ++} ++ ++/* ++ * Since this CPU is going 'away' for a while, fold any nr_active delta ++ * we might have. Assumes we're called after migrate_tasks() so that the ++ * nr_active count is stable. ++ * ++ * Also see the comment "Global load-average calculations". ++ */ ++static void calc_load_migrate(struct rq *rq) ++{ ++ long delta = calc_load_fold_active(rq); ++ if (delta) ++ atomic_long_add(delta, &calc_load_tasks); ++} ++ ++static void put_prev_task_fake(struct rq *rq, struct task_struct *prev) ++{ ++} ++ ++static const struct sched_class fake_sched_class = { ++ .put_prev_task = put_prev_task_fake, ++}; ++ ++static struct task_struct fake_task = { ++ /* ++ * Avoid pull_{rt,dl}_task() ++ */ ++ .prio = MAX_PRIO + 1, ++ .sched_class = &fake_sched_class, ++}; ++ ++/* ++ * Migrate all tasks from the rq, sleeping tasks will be migrated by ++ * try_to_wake_up()->select_task_rq(). ++ * ++ * Called with rq->lock held even though we'er in stop_machine() and ++ * there's no concurrency possible, we hold the required locks anyway ++ * because of lock validation efforts. ++ */ ++static void migrate_tasks(unsigned int dead_cpu) ++{ ++ struct rq *rq = cpu_rq(dead_cpu); ++ struct task_struct *next, *stop = rq->stop; ++ int dest_cpu; ++ ++ /* ++ * Fudge the rq selection such that the below task selection loop ++ * doesn't get stuck on the currently eligible stop task. ++ * ++ * We're currently inside stop_machine() and the rq is either stuck ++ * in the stop_machine_cpu_stop() loop, or we're executing this code, ++ * either way we should never end up calling schedule() until we're ++ * done here. ++ */ ++ rq->stop = NULL; ++ ++ /* ++ * put_prev_task() and pick_next_task() sched ++ * class method both need to have an up-to-date ++ * value of rq->clock[_task] ++ */ ++ update_rq_clock(rq); ++ ++ for ( ; ; ) { ++ /* ++ * There's this thread running, bail when that's the only ++ * remaining thread. ++ */ ++ if (rq->nr_running == 1) ++ break; ++ ++ next = pick_next_task(rq, &fake_task); ++ BUG_ON(!next); ++ next->sched_class->put_prev_task(rq, next); ++ ++ /* Find suitable destination for @next, with force if needed. */ ++ dest_cpu = select_fallback_rq(dead_cpu, next); ++ raw_spin_unlock(&rq->lock); ++ ++ __migrate_task(next, dead_cpu, dest_cpu); ++ ++ raw_spin_lock(&rq->lock); ++ } ++ ++ rq->stop = stop; ++} ++ ++#endif /* CONFIG_HOTPLUG_CPU */ ++ ++#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL) ++ ++static struct ctl_table sd_ctl_dir[] = { ++ { ++ .procname = "sched_domain", ++ .mode = 0555, ++ }, ++ {} ++}; ++ ++static struct ctl_table sd_ctl_root[] = { ++ { ++ .procname = "kernel", ++ .mode = 0555, ++ .child = sd_ctl_dir, ++ }, ++ {} ++}; ++ ++static struct ctl_table *sd_alloc_ctl_entry(int n) ++{ ++ struct ctl_table *entry = ++ kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL); ++ ++ return entry; ++} ++ ++static void sd_free_ctl_entry(struct ctl_table **tablep) ++{ ++ struct ctl_table *entry; ++ ++ /* ++ * In the intermediate directories, both the child directory and ++ * procname are dynamically allocated and could fail but the mode ++ * will always be set. In the lowest directory the names are ++ * static strings and all have proc handlers. ++ */ ++ for (entry = *tablep; entry->mode; entry++) { ++ if (entry->child) ++ sd_free_ctl_entry(&entry->child); ++ if (entry->proc_handler == NULL) ++ kfree(entry->procname); ++ } ++ ++ kfree(*tablep); ++ *tablep = NULL; ++} ++ ++static int min_load_idx = 0; ++static int max_load_idx = CPU_LOAD_IDX_MAX-1; ++ ++static void ++set_table_entry(struct ctl_table *entry, ++ const char *procname, void *data, int maxlen, ++ umode_t mode, proc_handler *proc_handler, ++ bool load_idx) ++{ ++ entry->procname = procname; ++ entry->data = data; ++ entry->maxlen = maxlen; ++ entry->mode = mode; ++ entry->proc_handler = proc_handler; ++ ++ if (load_idx) { ++ entry->extra1 = &min_load_idx; ++ entry->extra2 = &max_load_idx; ++ } ++} ++ ++static struct ctl_table * ++sd_alloc_ctl_domain_table(struct sched_domain *sd) ++{ ++ struct ctl_table *table = sd_alloc_ctl_entry(14); ++ ++ if (table == NULL) ++ return NULL; ++ ++ set_table_entry(&table[0], "min_interval", &sd->min_interval, ++ sizeof(long), 0644, proc_doulongvec_minmax, false); ++ set_table_entry(&table[1], "max_interval", &sd->max_interval, ++ sizeof(long), 0644, proc_doulongvec_minmax, false); ++ set_table_entry(&table[2], "busy_idx", &sd->busy_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[3], "idle_idx", &sd->idle_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[5], "wake_idx", &sd->wake_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[7], "busy_factor", &sd->busy_factor, ++ sizeof(int), 0644, proc_dointvec_minmax, false); ++ set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct, ++ sizeof(int), 0644, proc_dointvec_minmax, false); ++ set_table_entry(&table[9], "cache_nice_tries", ++ &sd->cache_nice_tries, ++ sizeof(int), 0644, proc_dointvec_minmax, false); ++ set_table_entry(&table[10], "flags", &sd->flags, ++ sizeof(int), 0644, proc_dointvec_minmax, false); ++ set_table_entry(&table[11], "max_newidle_lb_cost", ++ &sd->max_newidle_lb_cost, ++ sizeof(long), 0644, proc_doulongvec_minmax, false); ++ set_table_entry(&table[12], "name", sd->name, ++ CORENAME_MAX_SIZE, 0444, proc_dostring, false); ++ /* &table[13] is terminator */ ++ ++ return table; ++} ++ ++static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu) ++{ ++ struct ctl_table *entry, *table; ++ struct sched_domain *sd; ++ int domain_num = 0, i; ++ char buf[32]; ++ ++ for_each_domain(cpu, sd) ++ domain_num++; ++ entry = table = sd_alloc_ctl_entry(domain_num + 1); ++ if (table == NULL) ++ return NULL; ++ ++ i = 0; ++ for_each_domain(cpu, sd) { ++ snprintf(buf, 32, "domain%d", i); ++ entry->procname = kstrdup(buf, GFP_KERNEL); ++ entry->mode = 0555; ++ entry->child = sd_alloc_ctl_domain_table(sd); ++ entry++; ++ i++; ++ } ++ return table; ++} ++ ++static struct ctl_table_header *sd_sysctl_header; ++static void register_sched_domain_sysctl(void) ++{ ++ int i, cpu_num = num_possible_cpus(); ++ struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1); ++ char buf[32]; ++ ++ WARN_ON(sd_ctl_dir[0].child); ++ sd_ctl_dir[0].child = entry; ++ ++ if (entry == NULL) ++ return; ++ ++ for_each_possible_cpu(i) { ++ snprintf(buf, 32, "cpu%d", i); ++ entry->procname = kstrdup(buf, GFP_KERNEL); ++ entry->mode = 0555; ++ entry->child = sd_alloc_ctl_cpu_table(i); ++ entry++; ++ } ++ ++ WARN_ON(sd_sysctl_header); ++ sd_sysctl_header = register_sysctl_table(sd_ctl_root); ++} ++ ++/* may be called multiple times per register */ ++static void unregister_sched_domain_sysctl(void) ++{ ++ if (sd_sysctl_header) ++ unregister_sysctl_table(sd_sysctl_header); ++ sd_sysctl_header = NULL; ++ if (sd_ctl_dir[0].child) ++ sd_free_ctl_entry(&sd_ctl_dir[0].child); ++} ++#else ++static void register_sched_domain_sysctl(void) ++{ ++} ++static void unregister_sched_domain_sysctl(void) ++{ ++} ++#endif ++ ++static void set_rq_online(struct rq *rq) ++{ ++ if (!rq->online) { ++ const struct sched_class *class; ++ ++ cpumask_set_cpu(rq->cpu, rq->rd->online); ++ rq->online = 1; ++ ++ for_each_class(class) { ++ if (class->rq_online) ++ class->rq_online(rq); ++ } ++ } ++} ++ ++static void set_rq_offline(struct rq *rq) ++{ ++ if (rq->online) { ++ const struct sched_class *class; ++ ++ for_each_class(class) { ++ if (class->rq_offline) ++ class->rq_offline(rq); ++ } ++ ++ cpumask_clear_cpu(rq->cpu, rq->rd->online); ++ rq->online = 0; ++ } ++} ++ ++/* ++ * migration_call - callback that gets triggered when a CPU is added. ++ * Here we can start up the necessary migration thread for the new CPU. ++ */ ++static int ++migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) ++{ ++ int cpu = (long)hcpu; ++ unsigned long flags; ++ struct rq *rq = cpu_rq(cpu); ++ ++ switch (action & ~CPU_TASKS_FROZEN) { ++ ++ case CPU_UP_PREPARE: ++ rq->calc_load_update = calc_load_update; ++ break; ++ ++ case CPU_ONLINE: ++ /* Update our root-domain */ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ if (rq->rd) { ++ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); ++ ++ set_rq_online(rq); ++ } ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ break; ++ ++#ifdef CONFIG_HOTPLUG_CPU ++ case CPU_DYING: ++ sched_ttwu_pending(); ++ /* Update our root-domain */ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ if (rq->rd) { ++ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); ++ set_rq_offline(rq); ++ } ++ migrate_tasks(cpu); ++ BUG_ON(rq->nr_running != 1); /* the migration thread */ ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ break; ++ ++ case CPU_DEAD: ++ calc_load_migrate(rq); ++ break; ++#endif ++ } ++ ++ update_max_interval(); ++ ++ return NOTIFY_OK; ++} ++ ++/* ++ * Register at high priority so that task migration (migrate_all_tasks) ++ * happens before everything else. This has to be lower priority than ++ * the notifier in the perf_event subsystem, though. ++ */ ++static struct notifier_block migration_notifier = { ++ .notifier_call = migration_call, ++ .priority = CPU_PRI_MIGRATION, ++}; ++ ++static void __cpuinit set_cpu_rq_start_time(void) ++{ ++ int cpu = smp_processor_id(); ++ struct rq *rq = cpu_rq(cpu); ++ rq->age_stamp = sched_clock_cpu(cpu); ++} ++ ++static int sched_cpu_active(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) ++{ ++ switch (action & ~CPU_TASKS_FROZEN) { ++ case CPU_STARTING: ++ set_cpu_rq_start_time(); ++ return NOTIFY_OK; ++ case CPU_ONLINE: ++ /* ++ * At this point a starting CPU has marked itself as online via ++ * set_cpu_online(). But it might not yet have marked itself ++ * as active, which is essential from here on. ++ * ++ * Thus, fall-through and help the starting CPU along. ++ */ ++ case CPU_DOWN_FAILED: ++ set_cpu_active((long)hcpu, true); ++ return NOTIFY_OK; ++ default: ++ return NOTIFY_DONE; ++ } ++} ++ ++static int sched_cpu_inactive(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) ++{ ++ switch (action & ~CPU_TASKS_FROZEN) { ++ case CPU_DOWN_PREPARE: ++ set_cpu_active((long)hcpu, false); ++ return NOTIFY_OK; ++ default: ++ return NOTIFY_DONE; ++ } ++} ++ ++static int __init migration_init(void) ++{ ++ void *cpu = (void *)(long)smp_processor_id(); ++ int err; ++ ++ /* Initialize migration for the boot CPU */ ++ err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu); ++ BUG_ON(err == NOTIFY_BAD); ++ migration_call(&migration_notifier, CPU_ONLINE, cpu); ++ register_cpu_notifier(&migration_notifier); ++ ++ /* Register cpu active notifiers */ ++ cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE); ++ cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE); ++ ++ return 0; ++} ++early_initcall(migration_init); ++#endif ++ ++#ifdef CONFIG_SMP ++ ++static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */ ++ ++#ifdef CONFIG_SCHED_DEBUG ++ ++static __read_mostly int sched_debug_enabled; ++ ++static int __init sched_debug_setup(char *str) ++{ ++ sched_debug_enabled = 1; ++ ++ return 0; ++} ++early_param("sched_debug", sched_debug_setup); ++ ++static inline bool sched_debug(void) ++{ ++ return sched_debug_enabled; ++} ++ ++static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, ++ struct cpumask *groupmask) ++{ ++ struct sched_group *group = sd->groups; ++ ++ cpumask_clear(groupmask); ++ ++ printk(KERN_DEBUG "%*s domain %d: ", level, "", level); ++ ++ if (!(sd->flags & SD_LOAD_BALANCE)) { ++ printk("does not load-balance\n"); ++ if (sd->parent) ++ printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain" ++ " has parent"); ++ return -1; ++ } ++ ++ printk(KERN_CONT "span %*pbl level %s\n", ++ cpumask_pr_args(sched_domain_span(sd)), sd->name); ++ ++ if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) { ++ printk(KERN_ERR "ERROR: domain->span does not contain " ++ "CPU%d\n", cpu); ++ } ++ if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) { ++ printk(KERN_ERR "ERROR: domain->groups does not contain" ++ " CPU%d\n", cpu); ++ } ++ ++ printk(KERN_DEBUG "%*s groups:", level + 1, ""); ++ do { ++ if (!group) { ++ printk("\n"); ++ printk(KERN_ERR "ERROR: group is NULL\n"); ++ break; ++ } ++ ++ if (!cpumask_weight(sched_group_cpus(group))) { ++ printk(KERN_CONT "\n"); ++ printk(KERN_ERR "ERROR: empty group\n"); ++ break; ++ } ++ ++ if (!(sd->flags & SD_OVERLAP) && ++ cpumask_intersects(groupmask, sched_group_cpus(group))) { ++ printk(KERN_CONT "\n"); ++ printk(KERN_ERR "ERROR: repeated CPUs\n"); ++ break; ++ } ++ ++ cpumask_or(groupmask, groupmask, sched_group_cpus(group)); ++ ++ printk(KERN_CONT " %*pbl", ++ cpumask_pr_args(sched_group_cpus(group))); ++ if (group->sgc->capacity != SCHED_CAPACITY_SCALE) { ++ printk(KERN_CONT " (cpu_capacity = %d)", ++ group->sgc->capacity); ++ } ++ ++ group = group->next; ++ } while (group != sd->groups); ++ printk(KERN_CONT "\n"); ++ ++ if (!cpumask_equal(sched_domain_span(sd), groupmask)) ++ printk(KERN_ERR "ERROR: groups don't span domain->span\n"); ++ ++ if (sd->parent && ++ !cpumask_subset(groupmask, sched_domain_span(sd->parent))) ++ printk(KERN_ERR "ERROR: parent span is not a superset " ++ "of domain->span\n"); ++ return 0; ++} ++ ++static void sched_domain_debug(struct sched_domain *sd, int cpu) ++{ ++ int level = 0; ++ ++ if (!sched_debug_enabled) ++ return; ++ ++ if (!sd) { ++ printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); ++ return; ++ } ++ ++ printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); ++ ++ for (;;) { ++ if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask)) ++ break; ++ level++; ++ sd = sd->parent; ++ if (!sd) ++ break; ++ } ++} ++#else /* !CONFIG_SCHED_DEBUG */ ++# define sched_domain_debug(sd, cpu) do { } while (0) ++static inline bool sched_debug(void) ++{ ++ return false; ++} ++#endif /* CONFIG_SCHED_DEBUG */ ++ ++static int sd_degenerate(struct sched_domain *sd) ++{ ++ if (cpumask_weight(sched_domain_span(sd)) == 1) ++ return 1; ++ ++ /* Following flags need at least 2 groups */ ++ if (sd->flags & (SD_LOAD_BALANCE | ++ SD_BALANCE_NEWIDLE | ++ SD_BALANCE_FORK | ++ SD_BALANCE_EXEC | ++ SD_SHARE_CPUCAPACITY | ++ SD_SHARE_PKG_RESOURCES | ++ SD_SHARE_POWERDOMAIN)) { ++ if (sd->groups != sd->groups->next) ++ return 0; ++ } ++ ++ /* Following flags don't use groups */ ++ if (sd->flags & (SD_WAKE_AFFINE)) ++ return 0; ++ ++ return 1; ++} ++ ++static int ++sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) ++{ ++ unsigned long cflags = sd->flags, pflags = parent->flags; ++ ++ if (sd_degenerate(parent)) ++ return 1; ++ ++ if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent))) ++ return 0; ++ ++ /* Flags needing groups don't count if only 1 group in parent */ ++ if (parent->groups == parent->groups->next) { ++ pflags &= ~(SD_LOAD_BALANCE | ++ SD_BALANCE_NEWIDLE | ++ SD_BALANCE_FORK | ++ SD_BALANCE_EXEC | ++ SD_SHARE_CPUCAPACITY | ++ SD_SHARE_PKG_RESOURCES | ++ SD_PREFER_SIBLING | ++ SD_SHARE_POWERDOMAIN); ++ if (nr_node_ids == 1) ++ pflags &= ~SD_SERIALIZE; ++ } ++ if (~cflags & pflags) ++ return 0; ++ ++ return 1; ++} ++ ++static void free_rootdomain(struct rcu_head *rcu) ++{ ++ struct root_domain *rd = container_of(rcu, struct root_domain, rcu); ++ ++ cpupri_cleanup(&rd->cpupri); ++ cpudl_cleanup(&rd->cpudl); ++ free_cpumask_var(rd->dlo_mask); ++ free_cpumask_var(rd->rto_mask); ++ free_cpumask_var(rd->online); ++ free_cpumask_var(rd->span); ++ kfree(rd); ++} ++ ++static void rq_attach_root(struct rq *rq, struct root_domain *rd) ++{ ++ struct root_domain *old_rd = NULL; ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ ++ if (rq->rd) { ++ old_rd = rq->rd; ++ ++ if (cpumask_test_cpu(rq->cpu, old_rd->online)) ++ set_rq_offline(rq); ++ ++ cpumask_clear_cpu(rq->cpu, old_rd->span); ++ ++ /* ++ * If we dont want to free the old_rd yet then ++ * set old_rd to NULL to skip the freeing later ++ * in this function: ++ */ ++ if (!atomic_dec_and_test(&old_rd->refcount)) ++ old_rd = NULL; ++ } ++ ++ atomic_inc(&rd->refcount); ++ rq->rd = rd; ++ ++ cpumask_set_cpu(rq->cpu, rd->span); ++ if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) ++ set_rq_online(rq); ++ ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ ++ if (old_rd) ++ call_rcu_sched(&old_rd->rcu, free_rootdomain); ++} ++ ++static int init_rootdomain(struct root_domain *rd) ++{ ++ memset(rd, 0, sizeof(*rd)); ++ ++ if (!alloc_cpumask_var(&rd->span, GFP_KERNEL)) ++ goto out; ++ if (!alloc_cpumask_var(&rd->online, GFP_KERNEL)) ++ goto free_span; ++ if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL)) ++ goto free_online; ++ if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) ++ goto free_dlo_mask; ++ ++ init_dl_bw(&rd->dl_bw); ++ if (cpudl_init(&rd->cpudl) != 0) ++ goto free_dlo_mask; ++ ++ if (cpupri_init(&rd->cpupri) != 0) ++ goto free_rto_mask; ++ return 0; ++ ++free_rto_mask: ++ free_cpumask_var(rd->rto_mask); ++free_dlo_mask: ++ free_cpumask_var(rd->dlo_mask); ++free_online: ++ free_cpumask_var(rd->online); ++free_span: ++ free_cpumask_var(rd->span); ++out: ++ return -ENOMEM; ++} ++ ++/* ++ * By default the system creates a single root-domain with all cpus as ++ * members (mimicking the global state we have today). ++ */ ++struct root_domain def_root_domain; ++ ++static void init_defrootdomain(void) ++{ ++ init_rootdomain(&def_root_domain); ++ ++ atomic_set(&def_root_domain.refcount, 1); ++} ++ ++static struct root_domain *alloc_rootdomain(void) ++{ ++ struct root_domain *rd; ++ ++ rd = kmalloc(sizeof(*rd), GFP_KERNEL); ++ if (!rd) ++ return NULL; ++ ++ if (init_rootdomain(rd) != 0) { ++ kfree(rd); ++ return NULL; ++ } ++ ++ return rd; ++} ++ ++static void free_sched_groups(struct sched_group *sg, int free_sgc) ++{ ++ struct sched_group *tmp, *first; ++ ++ if (!sg) ++ return; ++ ++ first = sg; ++ do { ++ tmp = sg->next; ++ ++ if (free_sgc && atomic_dec_and_test(&sg->sgc->ref)) ++ kfree(sg->sgc); ++ ++ kfree(sg); ++ sg = tmp; ++ } while (sg != first); ++} ++ ++static void free_sched_domain(struct rcu_head *rcu) ++{ ++ struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu); ++ ++ /* ++ * If its an overlapping domain it has private groups, iterate and ++ * nuke them all. ++ */ ++ if (sd->flags & SD_OVERLAP) { ++ free_sched_groups(sd->groups, 1); ++ } else if (atomic_dec_and_test(&sd->groups->ref)) { ++ kfree(sd->groups->sgc); ++ kfree(sd->groups); ++ } ++ kfree(sd); ++} ++ ++static void destroy_sched_domain(struct sched_domain *sd, int cpu) ++{ ++ call_rcu(&sd->rcu, free_sched_domain); ++} ++ ++static void destroy_sched_domains(struct sched_domain *sd, int cpu) ++{ ++ for (; sd; sd = sd->parent) ++ destroy_sched_domain(sd, cpu); ++} ++ ++/* ++ * Keep a special pointer to the highest sched_domain that has ++ * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this ++ * allows us to avoid some pointer chasing select_idle_sibling(). ++ * ++ * Also keep a unique ID per domain (we use the first cpu number in ++ * the cpumask of the domain), this allows us to quickly tell if ++ * two cpus are in the same cache domain, see cpus_share_cache(). ++ */ ++DEFINE_PER_CPU(struct sched_domain *, sd_llc); ++DEFINE_PER_CPU(int, sd_llc_size); ++DEFINE_PER_CPU(int, sd_llc_id); ++DEFINE_PER_CPU(struct sched_domain *, sd_numa); ++DEFINE_PER_CPU(struct sched_domain *, sd_busy); ++DEFINE_PER_CPU(struct sched_domain *, sd_asym); ++ ++static void update_top_cache_domain(int cpu) ++{ ++ struct sched_domain *sd; ++ struct sched_domain *busy_sd = NULL; ++ int id = cpu; ++ int size = 1; ++ ++ sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES); ++ if (sd) { ++ id = cpumask_first(sched_domain_span(sd)); ++ size = cpumask_weight(sched_domain_span(sd)); ++ busy_sd = sd->parent; /* sd_busy */ ++ } ++ rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd); ++ ++ rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); ++ per_cpu(sd_llc_size, cpu) = size; ++ per_cpu(sd_llc_id, cpu) = id; ++ ++ sd = lowest_flag_domain(cpu, SD_NUMA); ++ rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); ++ ++ sd = highest_flag_domain(cpu, SD_ASYM_PACKING); ++ rcu_assign_pointer(per_cpu(sd_asym, cpu), sd); ++} ++ ++/* ++ * Attach the domain 'sd' to 'cpu' as its base domain. Callers must ++ * hold the hotplug lock. ++ */ ++static void ++cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ struct sched_domain *tmp; ++ ++ /* Remove the sched domains which do not contribute to scheduling. */ ++ for (tmp = sd; tmp; ) { ++ struct sched_domain *parent = tmp->parent; ++ if (!parent) ++ break; ++ ++ if (sd_parent_degenerate(tmp, parent)) { ++ tmp->parent = parent->parent; ++ if (parent->parent) ++ parent->parent->child = tmp; ++ /* ++ * Transfer SD_PREFER_SIBLING down in case of a ++ * degenerate parent; the spans match for this ++ * so the property transfers. ++ */ ++ if (parent->flags & SD_PREFER_SIBLING) ++ tmp->flags |= SD_PREFER_SIBLING; ++ destroy_sched_domain(parent, cpu); ++ } else ++ tmp = tmp->parent; ++ } ++ ++ if (sd && sd_degenerate(sd)) { ++ tmp = sd; ++ sd = sd->parent; ++ destroy_sched_domain(tmp, cpu); ++ if (sd) ++ sd->child = NULL; ++ } ++ ++ sched_domain_debug(sd, cpu); ++ ++ rq_attach_root(rq, rd); ++ tmp = rq->sd; ++ rcu_assign_pointer(rq->sd, sd); ++ destroy_sched_domains(tmp, cpu); ++ ++ update_top_cache_domain(cpu); ++} ++ ++/* Setup the mask of cpus configured for isolated domains */ ++static int __init isolated_cpu_setup(char *str) ++{ ++ alloc_bootmem_cpumask_var(&cpu_isolated_map); ++ cpulist_parse(str, cpu_isolated_map); ++ return 1; ++} ++ ++__setup("isolcpus=", isolated_cpu_setup); ++ ++struct s_data { ++ struct sched_domain ** __percpu sd; ++ struct root_domain *rd; ++}; ++ ++enum s_alloc { ++ sa_rootdomain, ++ sa_sd, ++ sa_sd_storage, ++ sa_none, ++}; ++ ++/* ++ * Build an iteration mask that can exclude certain CPUs from the upwards ++ * domain traversal. ++ * ++ * Asymmetric node setups can result in situations where the domain tree is of ++ * unequal depth, make sure to skip domains that already cover the entire ++ * range. ++ * ++ * In that case build_sched_domains() will have terminated the iteration early ++ * and our sibling sd spans will be empty. Domains should always include the ++ * cpu they're built on, so check that. ++ * ++ */ ++static void build_group_mask(struct sched_domain *sd, struct sched_group *sg) ++{ ++ const struct cpumask *span = sched_domain_span(sd); ++ struct sd_data *sdd = sd->private; ++ struct sched_domain *sibling; ++ int i; ++ ++ for_each_cpu(i, span) { ++ sibling = *per_cpu_ptr(sdd->sd, i); ++ if (!cpumask_test_cpu(i, sched_domain_span(sibling))) ++ continue; ++ ++ cpumask_set_cpu(i, sched_group_mask(sg)); ++ } ++} ++ ++/* ++ * Return the canonical balance cpu for this group, this is the first cpu ++ * of this group that's also in the iteration mask. ++ */ ++int group_balance_cpu(struct sched_group *sg) ++{ ++ return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg)); ++} ++ ++static int ++build_overlap_sched_groups(struct sched_domain *sd, int cpu) ++{ ++ struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg; ++ const struct cpumask *span = sched_domain_span(sd); ++ struct cpumask *covered = sched_domains_tmpmask; ++ struct sd_data *sdd = sd->private; ++ struct sched_domain *sibling; ++ int i; ++ ++ cpumask_clear(covered); ++ ++ for_each_cpu(i, span) { ++ struct cpumask *sg_span; ++ ++ if (cpumask_test_cpu(i, covered)) ++ continue; ++ ++ sibling = *per_cpu_ptr(sdd->sd, i); ++ ++ /* See the comment near build_group_mask(). */ ++ if (!cpumask_test_cpu(i, sched_domain_span(sibling))) ++ continue; ++ ++ sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), ++ GFP_KERNEL, cpu_to_node(cpu)); ++ ++ if (!sg) ++ goto fail; ++ ++ sg_span = sched_group_cpus(sg); ++ if (sibling->child) ++ cpumask_copy(sg_span, sched_domain_span(sibling->child)); ++ else ++ cpumask_set_cpu(i, sg_span); ++ ++ cpumask_or(covered, covered, sg_span); ++ ++ sg->sgc = *per_cpu_ptr(sdd->sgc, i); ++ if (atomic_inc_return(&sg->sgc->ref) == 1) ++ build_group_mask(sd, sg); ++ ++ /* ++ * Initialize sgc->capacity such that even if we mess up the ++ * domains and no possible iteration will get us here, we won't ++ * die on a /0 trap. ++ */ ++ sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); ++ ++ /* ++ * Make sure the first group of this domain contains the ++ * canonical balance cpu. Otherwise the sched_domain iteration ++ * breaks. See update_sg_lb_stats(). ++ */ ++ if ((!groups && cpumask_test_cpu(cpu, sg_span)) || ++ group_balance_cpu(sg) == cpu) ++ groups = sg; ++ ++ if (!first) ++ first = sg; ++ if (last) ++ last->next = sg; ++ last = sg; ++ last->next = first; ++ } ++ sd->groups = groups; ++ ++ return 0; ++ ++fail: ++ free_sched_groups(first, 0); ++ ++ return -ENOMEM; ++} ++ ++static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg) ++{ ++ struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); ++ struct sched_domain *child = sd->child; ++ ++ if (child) ++ cpu = cpumask_first(sched_domain_span(child)); ++ ++ if (sg) { ++ *sg = *per_cpu_ptr(sdd->sg, cpu); ++ (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu); ++ atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */ ++ } ++ ++ return cpu; ++} ++ ++/* ++ * build_sched_groups will build a circular linked list of the groups ++ * covered by the given span, and will set each group's ->cpumask correctly, ++ * and ->cpu_capacity to 0. ++ * ++ * Assumes the sched_domain tree is fully constructed ++ */ ++static int ++build_sched_groups(struct sched_domain *sd, int cpu) ++{ ++ struct sched_group *first = NULL, *last = NULL; ++ struct sd_data *sdd = sd->private; ++ const struct cpumask *span = sched_domain_span(sd); ++ struct cpumask *covered; ++ int i; ++ ++ get_group(cpu, sdd, &sd->groups); ++ atomic_inc(&sd->groups->ref); ++ ++ if (cpu != cpumask_first(span)) ++ return 0; ++ ++ lockdep_assert_held(&sched_domains_mutex); ++ covered = sched_domains_tmpmask; ++ ++ cpumask_clear(covered); ++ ++ for_each_cpu(i, span) { ++ struct sched_group *sg; ++ int group, j; ++ ++ if (cpumask_test_cpu(i, covered)) ++ continue; ++ ++ group = get_group(i, sdd, &sg); ++ cpumask_setall(sched_group_mask(sg)); ++ ++ for_each_cpu(j, span) { ++ if (get_group(j, sdd, NULL) != group) ++ continue; ++ ++ cpumask_set_cpu(j, covered); ++ cpumask_set_cpu(j, sched_group_cpus(sg)); ++ } ++ ++ if (!first) ++ first = sg; ++ if (last) ++ last->next = sg; ++ last = sg; ++ } ++ last->next = first; ++ ++ return 0; ++} ++ ++/* ++ * Initialize sched groups cpu_capacity. ++ * ++ * cpu_capacity indicates the capacity of sched group, which is used while ++ * distributing the load between different sched groups in a sched domain. ++ * Typically cpu_capacity for all the groups in a sched domain will be same ++ * unless there are asymmetries in the topology. If there are asymmetries, ++ * group having more cpu_capacity will pickup more load compared to the ++ * group having less cpu_capacity. ++ */ ++static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) ++{ ++ struct sched_group *sg = sd->groups; ++ ++ WARN_ON(!sg); ++ ++ do { ++ sg->group_weight = cpumask_weight(sched_group_cpus(sg)); ++ sg = sg->next; ++ } while (sg != sd->groups); ++ ++ if (cpu != group_balance_cpu(sg)) ++ return; ++ ++ update_group_capacity(sd, cpu); ++ atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight); ++} ++ ++/* ++ * Initializers for schedule domains ++ * Non-inlined to reduce accumulated stack pressure in build_sched_domains() ++ */ ++ ++static int default_relax_domain_level = -1; ++int sched_domain_level_max; ++ ++static int __init setup_relax_domain_level(char *str) ++{ ++ if (kstrtoint(str, 0, &default_relax_domain_level)) ++ pr_warn("Unable to set relax_domain_level\n"); ++ ++ return 1; ++} ++__setup("relax_domain_level=", setup_relax_domain_level); ++ ++static void set_domain_attribute(struct sched_domain *sd, ++ struct sched_domain_attr *attr) ++{ ++ int request; ++ ++ if (!attr || attr->relax_domain_level < 0) { ++ if (default_relax_domain_level < 0) ++ return; ++ else ++ request = default_relax_domain_level; ++ } else ++ request = attr->relax_domain_level; ++ if (request < sd->level) { ++ /* turn off idle balance on this domain */ ++ sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); ++ } else { ++ /* turn on idle balance on this domain */ ++ sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); ++ } ++} ++ ++static void __sdt_free(const struct cpumask *cpu_map); ++static int __sdt_alloc(const struct cpumask *cpu_map); ++ ++static void __free_domain_allocs(struct s_data *d, enum s_alloc what, ++ const struct cpumask *cpu_map) ++{ ++ switch (what) { ++ case sa_rootdomain: ++ if (!atomic_read(&d->rd->refcount)) ++ free_rootdomain(&d->rd->rcu); /* fall through */ ++ case sa_sd: ++ free_percpu(d->sd); /* fall through */ ++ case sa_sd_storage: ++ __sdt_free(cpu_map); /* fall through */ ++ case sa_none: ++ break; ++ } ++} ++ ++static enum s_alloc __visit_domain_allocation_hell(struct s_data *d, ++ const struct cpumask *cpu_map) ++{ ++ memset(d, 0, sizeof(*d)); ++ ++ if (__sdt_alloc(cpu_map)) ++ return sa_sd_storage; ++ d->sd = alloc_percpu(struct sched_domain *); ++ if (!d->sd) ++ return sa_sd_storage; ++ d->rd = alloc_rootdomain(); ++ if (!d->rd) ++ return sa_sd; ++ return sa_rootdomain; ++} ++ ++/* ++ * NULL the sd_data elements we've used to build the sched_domain and ++ * sched_group structure so that the subsequent __free_domain_allocs() ++ * will not free the data we're using. ++ */ ++static void claim_allocations(int cpu, struct sched_domain *sd) ++{ ++ struct sd_data *sdd = sd->private; ++ ++ WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd); ++ *per_cpu_ptr(sdd->sd, cpu) = NULL; ++ ++ if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) ++ *per_cpu_ptr(sdd->sg, cpu) = NULL; ++ ++ if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref)) ++ *per_cpu_ptr(sdd->sgc, cpu) = NULL; ++} ++ ++#ifdef CONFIG_NUMA ++static int sched_domains_numa_levels; ++enum numa_topology_type sched_numa_topology_type; ++static int *sched_domains_numa_distance; ++int sched_max_numa_distance; ++static struct cpumask ***sched_domains_numa_masks; ++static int sched_domains_curr_level; ++#endif ++ ++/* ++ * SD_flags allowed in topology descriptions. ++ * ++ * SD_SHARE_CPUCAPACITY - describes SMT topologies ++ * SD_SHARE_PKG_RESOURCES - describes shared caches ++ * SD_NUMA - describes NUMA topologies ++ * SD_SHARE_POWERDOMAIN - describes shared power domain ++ * ++ * Odd one out: ++ * SD_ASYM_PACKING - describes SMT quirks ++ */ ++#define TOPOLOGY_SD_FLAGS \ ++ (SD_SHARE_CPUCAPACITY | \ ++ SD_SHARE_PKG_RESOURCES | \ ++ SD_NUMA | \ ++ SD_ASYM_PACKING | \ ++ SD_SHARE_POWERDOMAIN) ++ ++static struct sched_domain * ++sd_init(struct sched_domain_topology_level *tl, int cpu) ++{ ++ struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); ++ int sd_weight, sd_flags = 0; ++ ++#ifdef CONFIG_NUMA ++ /* ++ * Ugly hack to pass state to sd_numa_mask()... ++ */ ++ sched_domains_curr_level = tl->numa_level; ++#endif ++ ++ sd_weight = cpumask_weight(tl->mask(cpu)); ++ ++ if (tl->sd_flags) ++ sd_flags = (*tl->sd_flags)(); ++ if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS, ++ "wrong sd_flags in topology description\n")) ++ sd_flags &= ~TOPOLOGY_SD_FLAGS; ++ ++ *sd = (struct sched_domain){ ++ .min_interval = sd_weight, ++ .max_interval = 2*sd_weight, ++ .busy_factor = 32, ++ .imbalance_pct = 125, ++ ++ .cache_nice_tries = 0, ++ .busy_idx = 0, ++ .idle_idx = 0, ++ .newidle_idx = 0, ++ .wake_idx = 0, ++ .forkexec_idx = 0, ++ ++ .flags = 1*SD_LOAD_BALANCE ++ | 1*SD_BALANCE_NEWIDLE ++ | 1*SD_BALANCE_EXEC ++ | 1*SD_BALANCE_FORK ++ | 0*SD_BALANCE_WAKE ++ | 1*SD_WAKE_AFFINE ++ | 0*SD_SHARE_CPUCAPACITY ++ | 0*SD_SHARE_PKG_RESOURCES ++ | 0*SD_SERIALIZE ++ | 0*SD_PREFER_SIBLING ++ | 0*SD_NUMA ++ | sd_flags ++ , ++ ++ .last_balance = jiffies, ++ .balance_interval = sd_weight, ++ .smt_gain = 0, ++ .max_newidle_lb_cost = 0, ++ .next_decay_max_lb_cost = jiffies, ++#ifdef CONFIG_SCHED_DEBUG ++ .name = tl->name, ++#endif ++ }; ++ ++ /* ++ * Convert topological properties into behaviour. ++ */ ++ ++ if (sd->flags & SD_SHARE_CPUCAPACITY) { ++ sd->flags |= SD_PREFER_SIBLING; ++ sd->imbalance_pct = 110; ++ sd->smt_gain = 1178; /* ~15% */ ++ ++ } else if (sd->flags & SD_SHARE_PKG_RESOURCES) { ++ sd->imbalance_pct = 117; ++ sd->cache_nice_tries = 1; ++ sd->busy_idx = 2; ++ ++#ifdef CONFIG_NUMA ++ } else if (sd->flags & SD_NUMA) { ++ sd->cache_nice_tries = 2; ++ sd->busy_idx = 3; ++ sd->idle_idx = 2; ++ ++ sd->flags |= SD_SERIALIZE; ++ if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) { ++ sd->flags &= ~(SD_BALANCE_EXEC | ++ SD_BALANCE_FORK | ++ SD_WAKE_AFFINE); ++ } ++ ++#endif ++ } else { ++ sd->flags |= SD_PREFER_SIBLING; ++ sd->cache_nice_tries = 1; ++ sd->busy_idx = 2; ++ sd->idle_idx = 1; ++ } ++ ++ sd->private = &tl->data; ++ ++ return sd; ++} ++ ++/* ++ * Topology list, bottom-up. ++ */ ++static struct sched_domain_topology_level default_topology[] = { ++#ifdef CONFIG_SCHED_SMT ++ { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, ++#endif ++#ifdef CONFIG_SCHED_MC ++ { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, ++#endif ++ { cpu_cpu_mask, SD_INIT_NAME(DIE) }, ++ { NULL, }, ++}; ++ ++struct sched_domain_topology_level *sched_domain_topology = default_topology; ++ ++#define for_each_sd_topology(tl) \ ++ for (tl = sched_domain_topology; tl->mask; tl++) ++ ++void set_sched_topology(struct sched_domain_topology_level *tl) ++{ ++ sched_domain_topology = tl; ++} ++ ++#ifdef CONFIG_NUMA ++ ++static const struct cpumask *sd_numa_mask(int cpu) ++{ ++ return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)]; ++} ++ ++static void sched_numa_warn(const char *str) ++{ ++ static int done = false; ++ int i,j; ++ ++ if (done) ++ return; ++ ++ done = true; ++ ++ printk(KERN_WARNING "ERROR: %s\n\n", str); ++ ++ for (i = 0; i < nr_node_ids; i++) { ++ printk(KERN_WARNING " "); ++ for (j = 0; j < nr_node_ids; j++) ++ printk(KERN_CONT "%02d ", node_distance(i,j)); ++ printk(KERN_CONT "\n"); ++ } ++ printk(KERN_WARNING "\n"); ++} ++ ++bool find_numa_distance(int distance) ++{ ++ int i; ++ ++ if (distance == node_distance(0, 0)) ++ return true; ++ ++ for (i = 0; i < sched_domains_numa_levels; i++) { ++ if (sched_domains_numa_distance[i] == distance) ++ return true; ++ } ++ ++ return false; ++} ++ ++/* ++ * A system can have three types of NUMA topology: ++ * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system ++ * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes ++ * NUMA_BACKPLANE: nodes can reach other nodes through a backplane ++ * ++ * The difference between a glueless mesh topology and a backplane ++ * topology lies in whether communication between not directly ++ * connected nodes goes through intermediary nodes (where programs ++ * could run), or through backplane controllers. This affects ++ * placement of programs. ++ * ++ * The type of topology can be discerned with the following tests: ++ * - If the maximum distance between any nodes is 1 hop, the system ++ * is directly connected. ++ * - If for two nodes A and B, located N > 1 hops away from each other, ++ * there is an intermediary node C, which is < N hops away from both ++ * nodes A and B, the system is a glueless mesh. ++ */ ++static void init_numa_topology_type(void) ++{ ++ int a, b, c, n; ++ ++ n = sched_max_numa_distance; ++ ++ if (n <= 1) ++ sched_numa_topology_type = NUMA_DIRECT; ++ ++ for_each_online_node(a) { ++ for_each_online_node(b) { ++ /* Find two nodes furthest removed from each other. */ ++ if (node_distance(a, b) < n) ++ continue; ++ ++ /* Is there an intermediary node between a and b? */ ++ for_each_online_node(c) { ++ if (node_distance(a, c) < n && ++ node_distance(b, c) < n) { ++ sched_numa_topology_type = ++ NUMA_GLUELESS_MESH; ++ return; ++ } ++ } ++ ++ sched_numa_topology_type = NUMA_BACKPLANE; ++ return; ++ } ++ } ++} ++ ++static void sched_init_numa(void) ++{ ++ int next_distance, curr_distance = node_distance(0, 0); ++ struct sched_domain_topology_level *tl; ++ int level = 0; ++ int i, j, k; ++ ++ sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL); ++ if (!sched_domains_numa_distance) ++ return; ++ ++ /* ++ * O(nr_nodes^2) deduplicating selection sort -- in order to find the ++ * unique distances in the node_distance() table. ++ * ++ * Assumes node_distance(0,j) includes all distances in ++ * node_distance(i,j) in order to avoid cubic time. ++ */ ++ next_distance = curr_distance; ++ for (i = 0; i < nr_node_ids; i++) { ++ for (j = 0; j < nr_node_ids; j++) { ++ for (k = 0; k < nr_node_ids; k++) { ++ int distance = node_distance(i, k); ++ ++ if (distance > curr_distance && ++ (distance < next_distance || ++ next_distance == curr_distance)) ++ next_distance = distance; ++ ++ /* ++ * While not a strong assumption it would be nice to know ++ * about cases where if node A is connected to B, B is not ++ * equally connected to A. ++ */ ++ if (sched_debug() && node_distance(k, i) != distance) ++ sched_numa_warn("Node-distance not symmetric"); ++ ++ if (sched_debug() && i && !find_numa_distance(distance)) ++ sched_numa_warn("Node-0 not representative"); ++ } ++ if (next_distance != curr_distance) { ++ sched_domains_numa_distance[level++] = next_distance; ++ sched_domains_numa_levels = level; ++ curr_distance = next_distance; ++ } else break; ++ } ++ ++ /* ++ * In case of sched_debug() we verify the above assumption. ++ */ ++ if (!sched_debug()) ++ break; ++ } ++ ++ if (!level) ++ return; ++ ++ /* ++ * 'level' contains the number of unique distances, excluding the ++ * identity distance node_distance(i,i). ++ * ++ * The sched_domains_numa_distance[] array includes the actual distance ++ * numbers. ++ */ ++ ++ /* ++ * Here, we should temporarily reset sched_domains_numa_levels to 0. ++ * If it fails to allocate memory for array sched_domains_numa_masks[][], ++ * the array will contain less then 'level' members. This could be ++ * dangerous when we use it to iterate array sched_domains_numa_masks[][] ++ * in other functions. ++ * ++ * We reset it to 'level' at the end of this function. ++ */ ++ sched_domains_numa_levels = 0; ++ ++ sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL); ++ if (!sched_domains_numa_masks) ++ return; ++ ++ /* ++ * Now for each level, construct a mask per node which contains all ++ * cpus of nodes that are that many hops away from us. ++ */ ++ for (i = 0; i < level; i++) { ++ sched_domains_numa_masks[i] = ++ kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL); ++ if (!sched_domains_numa_masks[i]) ++ return; ++ ++ for (j = 0; j < nr_node_ids; j++) { ++ struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL); ++ if (!mask) ++ return; ++ ++ sched_domains_numa_masks[i][j] = mask; ++ ++ for (k = 0; k < nr_node_ids; k++) { ++ if (node_distance(j, k) > sched_domains_numa_distance[i]) ++ continue; ++ ++ cpumask_or(mask, mask, cpumask_of_node(k)); ++ } ++ } ++ } ++ ++ /* Compute default topology size */ ++ for (i = 0; sched_domain_topology[i].mask; i++); ++ ++ tl = kzalloc((i + level + 1) * ++ sizeof(struct sched_domain_topology_level), GFP_KERNEL); ++ if (!tl) ++ return; ++ ++ /* ++ * Copy the default topology bits.. ++ */ ++ for (i = 0; sched_domain_topology[i].mask; i++) ++ tl[i] = sched_domain_topology[i]; ++ ++ /* ++ * .. and append 'j' levels of NUMA goodness. ++ */ ++ for (j = 0; j < level; i++, j++) { ++ tl[i] = (struct sched_domain_topology_level){ ++ .mask = sd_numa_mask, ++ .sd_flags = cpu_numa_flags, ++ .flags = SDTL_OVERLAP, ++ .numa_level = j, ++ SD_INIT_NAME(NUMA) ++ }; ++ } ++ ++ sched_domain_topology = tl; ++ ++ sched_domains_numa_levels = level; ++ sched_max_numa_distance = sched_domains_numa_distance[level - 1]; ++ ++ init_numa_topology_type(); ++} ++ ++static void sched_domains_numa_masks_set(int cpu) ++{ ++ int i, j; ++ int node = cpu_to_node(cpu); ++ ++ for (i = 0; i < sched_domains_numa_levels; i++) { ++ for (j = 0; j < nr_node_ids; j++) { ++ if (node_distance(j, node) <= sched_domains_numa_distance[i]) ++ cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]); ++ } ++ } ++} ++ ++static void sched_domains_numa_masks_clear(int cpu) ++{ ++ int i, j; ++ for (i = 0; i < sched_domains_numa_levels; i++) { ++ for (j = 0; j < nr_node_ids; j++) ++ cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]); ++ } ++} ++ ++/* ++ * Update sched_domains_numa_masks[level][node] array when new cpus ++ * are onlined. ++ */ ++static int sched_domains_numa_masks_update(struct notifier_block *nfb, ++ unsigned long action, ++ void *hcpu) ++{ ++ int cpu = (long)hcpu; ++ ++ switch (action & ~CPU_TASKS_FROZEN) { ++ case CPU_ONLINE: ++ sched_domains_numa_masks_set(cpu); ++ break; ++ ++ case CPU_DEAD: ++ sched_domains_numa_masks_clear(cpu); ++ break; ++ ++ default: ++ return NOTIFY_DONE; ++ } ++ ++ return NOTIFY_OK; ++} ++#else ++static inline void sched_init_numa(void) ++{ ++} ++ ++static int sched_domains_numa_masks_update(struct notifier_block *nfb, ++ unsigned long action, ++ void *hcpu) ++{ ++ return 0; ++} ++#endif /* CONFIG_NUMA */ ++ ++static int __sdt_alloc(const struct cpumask *cpu_map) ++{ ++ struct sched_domain_topology_level *tl; ++ int j; ++ ++ for_each_sd_topology(tl) { ++ struct sd_data *sdd = &tl->data; ++ ++ sdd->sd = alloc_percpu(struct sched_domain *); ++ if (!sdd->sd) ++ return -ENOMEM; ++ ++ sdd->sg = alloc_percpu(struct sched_group *); ++ if (!sdd->sg) ++ return -ENOMEM; ++ ++ sdd->sgc = alloc_percpu(struct sched_group_capacity *); ++ if (!sdd->sgc) ++ return -ENOMEM; ++ ++ for_each_cpu(j, cpu_map) { ++ struct sched_domain *sd; ++ struct sched_group *sg; ++ struct sched_group_capacity *sgc; ++ ++ sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), ++ GFP_KERNEL, cpu_to_node(j)); ++ if (!sd) ++ return -ENOMEM; ++ ++ *per_cpu_ptr(sdd->sd, j) = sd; ++ ++ sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), ++ GFP_KERNEL, cpu_to_node(j)); ++ if (!sg) ++ return -ENOMEM; ++ ++ sg->next = sg; ++ ++ *per_cpu_ptr(sdd->sg, j) = sg; ++ ++ sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(), ++ GFP_KERNEL, cpu_to_node(j)); ++ if (!sgc) ++ return -ENOMEM; ++ ++ *per_cpu_ptr(sdd->sgc, j) = sgc; ++ } ++ } ++ ++ return 0; ++} ++ ++static void __sdt_free(const struct cpumask *cpu_map) ++{ ++ struct sched_domain_topology_level *tl; ++ int j; ++ ++ for_each_sd_topology(tl) { ++ struct sd_data *sdd = &tl->data; ++ ++ for_each_cpu(j, cpu_map) { ++ struct sched_domain *sd; ++ ++ if (sdd->sd) { ++ sd = *per_cpu_ptr(sdd->sd, j); ++ if (sd && (sd->flags & SD_OVERLAP)) ++ free_sched_groups(sd->groups, 0); ++ kfree(*per_cpu_ptr(sdd->sd, j)); ++ } ++ ++ if (sdd->sg) ++ kfree(*per_cpu_ptr(sdd->sg, j)); ++ if (sdd->sgc) ++ kfree(*per_cpu_ptr(sdd->sgc, j)); ++ } ++ free_percpu(sdd->sd); ++ sdd->sd = NULL; ++ free_percpu(sdd->sg); ++ sdd->sg = NULL; ++ free_percpu(sdd->sgc); ++ sdd->sgc = NULL; ++ } ++} ++ ++struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, ++ const struct cpumask *cpu_map, struct sched_domain_attr *attr, ++ struct sched_domain *child, int cpu) ++{ ++ struct sched_domain *sd = sd_init(tl, cpu); ++ if (!sd) ++ return child; ++ ++ cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu)); ++ if (child) { ++ sd->level = child->level + 1; ++ sched_domain_level_max = max(sched_domain_level_max, sd->level); ++ child->parent = sd; ++ sd->child = child; ++ ++ if (!cpumask_subset(sched_domain_span(child), ++ sched_domain_span(sd))) { ++ pr_err("BUG: arch topology borken\n"); ++#ifdef CONFIG_SCHED_DEBUG ++ pr_err(" the %s domain not a subset of the %s domain\n", ++ child->name, sd->name); ++#endif ++ /* Fixup, ensure @sd has at least @child cpus. */ ++ cpumask_or(sched_domain_span(sd), ++ sched_domain_span(sd), ++ sched_domain_span(child)); ++ } ++ ++ } ++ set_domain_attribute(sd, attr); ++ ++ return sd; ++} ++ ++/* ++ * Build sched domains for a given set of cpus and attach the sched domains ++ * to the individual cpus ++ */ ++static int build_sched_domains(const struct cpumask *cpu_map, ++ struct sched_domain_attr *attr) ++{ ++ enum s_alloc alloc_state; ++ struct sched_domain *sd; ++ struct s_data d; ++ int i, ret = -ENOMEM; ++ ++ alloc_state = __visit_domain_allocation_hell(&d, cpu_map); ++ if (alloc_state != sa_rootdomain) ++ goto error; ++ ++ /* Set up domains for cpus specified by the cpu_map. */ ++ for_each_cpu(i, cpu_map) { ++ struct sched_domain_topology_level *tl; ++ ++ sd = NULL; ++ for_each_sd_topology(tl) { ++ sd = build_sched_domain(tl, cpu_map, attr, sd, i); ++ if (tl == sched_domain_topology) ++ *per_cpu_ptr(d.sd, i) = sd; ++ if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP)) ++ sd->flags |= SD_OVERLAP; ++ if (cpumask_equal(cpu_map, sched_domain_span(sd))) ++ break; ++ } ++ } ++ ++ /* Build the groups for the domains */ ++ for_each_cpu(i, cpu_map) { ++ for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { ++ sd->span_weight = cpumask_weight(sched_domain_span(sd)); ++ if (sd->flags & SD_OVERLAP) { ++ if (build_overlap_sched_groups(sd, i)) ++ goto error; ++ } else { ++ if (build_sched_groups(sd, i)) ++ goto error; ++ } ++ } ++ } ++ ++ /* Calculate CPU capacity for physical packages and nodes */ ++ for (i = nr_cpumask_bits-1; i >= 0; i--) { ++ if (!cpumask_test_cpu(i, cpu_map)) ++ continue; ++ ++ for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { ++ claim_allocations(i, sd); ++ init_sched_groups_capacity(i, sd); ++ } ++ } ++ ++ /* Attach the domains */ ++ rcu_read_lock(); ++ for_each_cpu(i, cpu_map) { ++ sd = *per_cpu_ptr(d.sd, i); ++ cpu_attach_domain(sd, d.rd, i); ++ } ++ rcu_read_unlock(); ++ ++ ret = 0; ++error: ++ __free_domain_allocs(&d, alloc_state, cpu_map); ++ return ret; ++} ++ ++static cpumask_var_t *doms_cur; /* current sched domains */ ++static int ndoms_cur; /* number of sched domains in 'doms_cur' */ ++static struct sched_domain_attr *dattr_cur; ++ /* attribues of custom domains in 'doms_cur' */ ++ ++/* ++ * Special case: If a kmalloc of a doms_cur partition (array of ++ * cpumask) fails, then fallback to a single sched domain, ++ * as determined by the single cpumask fallback_doms. ++ */ ++static cpumask_var_t fallback_doms; ++ ++/* ++ * arch_update_cpu_topology lets virtualized architectures update the ++ * cpu core maps. It is supposed to return 1 if the topology changed ++ * or 0 if it stayed the same. ++ */ ++int __weak arch_update_cpu_topology(void) ++{ ++ return 0; ++} ++ ++cpumask_var_t *alloc_sched_domains(unsigned int ndoms) ++{ ++ int i; ++ cpumask_var_t *doms; ++ ++ doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL); ++ if (!doms) ++ return NULL; ++ for (i = 0; i < ndoms; i++) { ++ if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) { ++ free_sched_domains(doms, i); ++ return NULL; ++ } ++ } ++ return doms; ++} ++ ++void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms) ++{ ++ unsigned int i; ++ for (i = 0; i < ndoms; i++) ++ free_cpumask_var(doms[i]); ++ kfree(doms); ++} ++ ++/* ++ * Set up scheduler domains and groups. Callers must hold the hotplug lock. ++ * For now this just excludes isolated cpus, but could be used to ++ * exclude other special cases in the future. ++ */ ++static int init_sched_domains(const struct cpumask *cpu_map) ++{ ++ int err; ++ ++ arch_update_cpu_topology(); ++ ndoms_cur = 1; ++ doms_cur = alloc_sched_domains(ndoms_cur); ++ if (!doms_cur) ++ doms_cur = &fallback_doms; ++ cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map); ++ err = build_sched_domains(doms_cur[0], NULL); ++ register_sched_domain_sysctl(); ++ ++ return err; ++} ++ ++/* ++ * Detach sched domains from a group of cpus specified in cpu_map ++ * These cpus will now be attached to the NULL domain ++ */ ++static void detach_destroy_domains(const struct cpumask *cpu_map) ++{ ++ int i; ++ ++ rcu_read_lock(); ++ for_each_cpu(i, cpu_map) ++ cpu_attach_domain(NULL, &def_root_domain, i); ++ rcu_read_unlock(); ++} ++ ++/* handle null as "default" */ ++static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur, ++ struct sched_domain_attr *new, int idx_new) ++{ ++ struct sched_domain_attr tmp; ++ ++ /* fast path */ ++ if (!new && !cur) ++ return 1; ++ ++ tmp = SD_ATTR_INIT; ++ return !memcmp(cur ? (cur + idx_cur) : &tmp, ++ new ? (new + idx_new) : &tmp, ++ sizeof(struct sched_domain_attr)); ++} ++ ++/* ++ * Partition sched domains as specified by the 'ndoms_new' ++ * cpumasks in the array doms_new[] of cpumasks. This compares ++ * doms_new[] to the current sched domain partitioning, doms_cur[]. ++ * It destroys each deleted domain and builds each new domain. ++ * ++ * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'. ++ * The masks don't intersect (don't overlap.) We should setup one ++ * sched domain for each mask. CPUs not in any of the cpumasks will ++ * not be load balanced. If the same cpumask appears both in the ++ * current 'doms_cur' domains and in the new 'doms_new', we can leave ++ * it as it is. ++ * ++ * The passed in 'doms_new' should be allocated using ++ * alloc_sched_domains. This routine takes ownership of it and will ++ * free_sched_domains it when done with it. If the caller failed the ++ * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1, ++ * and partition_sched_domains() will fallback to the single partition ++ * 'fallback_doms', it also forces the domains to be rebuilt. ++ * ++ * If doms_new == NULL it will be replaced with cpu_online_mask. ++ * ndoms_new == 0 is a special case for destroying existing domains, ++ * and it will not create the default domain. ++ * ++ * Call with hotplug lock held ++ */ ++void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], ++ struct sched_domain_attr *dattr_new) ++{ ++ int i, j, n; ++ int new_topology; ++ ++ mutex_lock(&sched_domains_mutex); ++ ++ /* always unregister in case we don't destroy any domains */ ++ unregister_sched_domain_sysctl(); ++ ++ /* Let architecture update cpu core mappings. */ ++ new_topology = arch_update_cpu_topology(); ++ ++ n = doms_new ? ndoms_new : 0; ++ ++ /* Destroy deleted domains */ ++ for (i = 0; i < ndoms_cur; i++) { ++ for (j = 0; j < n && !new_topology; j++) { ++ if (cpumask_equal(doms_cur[i], doms_new[j]) ++ && dattrs_equal(dattr_cur, i, dattr_new, j)) ++ goto match1; ++ } ++ /* no match - a current sched domain not in new doms_new[] */ ++ detach_destroy_domains(doms_cur[i]); ++match1: ++ ; ++ } ++ ++ n = ndoms_cur; ++ if (doms_new == NULL) { ++ n = 0; ++ doms_new = &fallback_doms; ++ cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map); ++ WARN_ON_ONCE(dattr_new); ++ } ++ ++ /* Build new domains */ ++ for (i = 0; i < ndoms_new; i++) { ++ for (j = 0; j < n && !new_topology; j++) { ++ if (cpumask_equal(doms_new[i], doms_cur[j]) ++ && dattrs_equal(dattr_new, i, dattr_cur, j)) ++ goto match2; ++ } ++ /* no match - add a new doms_new */ ++ build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL); ++match2: ++ ; ++ } ++ ++ /* Remember the new sched domains */ ++ if (doms_cur != &fallback_doms) ++ free_sched_domains(doms_cur, ndoms_cur); ++ kfree(dattr_cur); /* kfree(NULL) is safe */ ++ doms_cur = doms_new; ++ dattr_cur = dattr_new; ++ ndoms_cur = ndoms_new; ++ ++ register_sched_domain_sysctl(); ++ ++ mutex_unlock(&sched_domains_mutex); ++} ++ ++static int num_cpus_frozen; /* used to mark begin/end of suspend/resume */ ++ ++/* ++ * Update cpusets according to cpu_active mask. If cpusets are ++ * disabled, cpuset_update_active_cpus() becomes a simple wrapper ++ * around partition_sched_domains(). ++ * ++ * If we come here as part of a suspend/resume, don't touch cpusets because we ++ * want to restore it back to its original state upon resume anyway. ++ */ ++static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action, ++ void *hcpu) ++{ ++ switch (action) { ++ case CPU_ONLINE_FROZEN: ++ case CPU_DOWN_FAILED_FROZEN: ++ ++ /* ++ * num_cpus_frozen tracks how many CPUs are involved in suspend ++ * resume sequence. As long as this is not the last online ++ * operation in the resume sequence, just build a single sched ++ * domain, ignoring cpusets. ++ */ ++ num_cpus_frozen--; ++ if (likely(num_cpus_frozen)) { ++ partition_sched_domains(1, NULL, NULL); ++ break; ++ } ++ ++ /* ++ * This is the last CPU online operation. So fall through and ++ * restore the original sched domains by considering the ++ * cpuset configurations. ++ */ ++ ++ case CPU_ONLINE: ++ cpuset_update_active_cpus(true); ++ break; ++ default: ++ return NOTIFY_DONE; ++ } ++ return NOTIFY_OK; ++} ++ ++static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action, ++ void *hcpu) ++{ ++ unsigned long flags; ++ long cpu = (long)hcpu; ++ struct dl_bw *dl_b; ++ bool overflow; ++ int cpus; ++ ++ switch (action) { ++ case CPU_DOWN_PREPARE: ++ rcu_read_lock_sched(); ++ dl_b = dl_bw_of(cpu); ++ ++ raw_spin_lock_irqsave(&dl_b->lock, flags); ++ cpus = dl_bw_cpus(cpu); ++ overflow = __dl_overflow(dl_b, cpus, 0, 0); ++ raw_spin_unlock_irqrestore(&dl_b->lock, flags); ++ ++ rcu_read_unlock_sched(); ++ ++ if (overflow) ++ return notifier_from_errno(-EBUSY); ++ cpuset_update_active_cpus(false); ++ break; ++ case CPU_DOWN_PREPARE_FROZEN: ++ num_cpus_frozen++; ++ partition_sched_domains(1, NULL, NULL); ++ break; ++ default: ++ return NOTIFY_DONE; ++ } ++ return NOTIFY_OK; ++} ++ ++void __init sched_init_smp(void) ++{ ++ cpumask_var_t non_isolated_cpus; ++ ++ alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL); ++ alloc_cpumask_var(&fallback_doms, GFP_KERNEL); ++ ++ sched_init_numa(); ++ ++ /* ++ * There's no userspace yet to cause hotplug operations; hence all the ++ * cpu masks are stable and all blatant races in the below code cannot ++ * happen. ++ */ ++ mutex_lock(&sched_domains_mutex); ++ init_sched_domains(cpu_active_mask); ++ cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map); ++ if (cpumask_empty(non_isolated_cpus)) ++ cpumask_set_cpu(smp_processor_id(), non_isolated_cpus); ++ mutex_unlock(&sched_domains_mutex); ++ ++ hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE); ++ hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE); ++ hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE); ++ ++ init_hrtick(); ++ ++ /* Move init over to a non-isolated CPU */ ++ if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0) ++ BUG(); ++ sched_init_granularity(); ++ free_cpumask_var(non_isolated_cpus); ++ ++ init_sched_rt_class(); ++ init_sched_dl_class(); ++} ++#else ++void __init sched_init_smp(void) ++{ ++ sched_init_granularity(); ++} ++#endif /* CONFIG_SMP */ ++ ++const_debug unsigned int sysctl_timer_migration = 1; ++ ++int in_sched_functions(unsigned long addr) ++{ ++ return in_lock_functions(addr) || ++ (addr >= (unsigned long)__sched_text_start ++ && addr < (unsigned long)__sched_text_end); ++} ++ ++#ifdef CONFIG_CGROUP_SCHED ++/* ++ * Default task group. ++ * Every task in system belongs to this group at bootup. ++ */ ++struct task_group root_task_group; ++LIST_HEAD(task_groups); ++#endif ++ ++DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); ++ ++void __init sched_init(void) ++{ ++ int i, j; ++ unsigned long alloc_size = 0, ptr; ++ ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ alloc_size += 2 * nr_cpu_ids * sizeof(void **); ++#endif ++#ifdef CONFIG_RT_GROUP_SCHED ++ alloc_size += 2 * nr_cpu_ids * sizeof(void **); ++#endif ++ if (alloc_size) { ++ ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT); ++ ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ root_task_group.se = (struct sched_entity **)ptr; ++ ptr += nr_cpu_ids * sizeof(void **); ++ ++ root_task_group.cfs_rq = (struct cfs_rq **)ptr; ++ ptr += nr_cpu_ids * sizeof(void **); ++ ++#endif /* CONFIG_FAIR_GROUP_SCHED */ ++#ifdef CONFIG_RT_GROUP_SCHED ++ root_task_group.rt_se = (struct sched_rt_entity **)ptr; ++ ptr += nr_cpu_ids * sizeof(void **); ++ ++ root_task_group.rt_rq = (struct rt_rq **)ptr; ++ ptr += nr_cpu_ids * sizeof(void **); ++ ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ } ++#ifdef CONFIG_CPUMASK_OFFSTACK ++ for_each_possible_cpu(i) { ++ per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node( ++ cpumask_size(), GFP_KERNEL, cpu_to_node(i)); ++ } ++#endif /* CONFIG_CPUMASK_OFFSTACK */ ++ ++ init_rt_bandwidth(&def_rt_bandwidth, ++ global_rt_period(), global_rt_runtime()); ++ init_dl_bandwidth(&def_dl_bandwidth, ++ global_rt_period(), global_rt_runtime()); ++ ++#ifdef CONFIG_SMP ++ init_defrootdomain(); ++#endif ++ ++#ifdef CONFIG_RT_GROUP_SCHED ++ init_rt_bandwidth(&root_task_group.rt_bandwidth, ++ global_rt_period(), global_rt_runtime()); ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ ++#ifdef CONFIG_CGROUP_SCHED ++ list_add(&root_task_group.list, &task_groups); ++ INIT_LIST_HEAD(&root_task_group.children); ++ INIT_LIST_HEAD(&root_task_group.siblings); ++ autogroup_init(&init_task); ++ ++#endif /* CONFIG_CGROUP_SCHED */ ++ ++ for_each_possible_cpu(i) { ++ struct rq *rq; ++ ++ rq = cpu_rq(i); ++ raw_spin_lock_init(&rq->lock); ++ rq->nr_running = 0; ++ rq->calc_load_active = 0; ++ rq->calc_load_update = jiffies + LOAD_FREQ; ++ init_cfs_rq(&rq->cfs); ++ init_rt_rq(&rq->rt); ++ init_dl_rq(&rq->dl); ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ root_task_group.shares = ROOT_TASK_GROUP_LOAD; ++ INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); ++ /* ++ * How much cpu bandwidth does root_task_group get? ++ * ++ * In case of task-groups formed thr' the cgroup filesystem, it ++ * gets 100% of the cpu resources in the system. This overall ++ * system cpu resource is divided among the tasks of ++ * root_task_group and its child task-groups in a fair manner, ++ * based on each entity's (task or task-group's) weight ++ * (se->load.weight). ++ * ++ * In other words, if root_task_group has 10 tasks of weight ++ * 1024) and two child groups A0 and A1 (of weight 1024 each), ++ * then A0's share of the cpu resource is: ++ * ++ * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% ++ * ++ * We achieve this by letting root_task_group's tasks sit ++ * directly in rq->cfs (i.e root_task_group->se[] = NULL). ++ */ ++ init_cfs_bandwidth(&root_task_group.cfs_bandwidth); ++ init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); ++#endif /* CONFIG_FAIR_GROUP_SCHED */ ++ ++ rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; ++#ifdef CONFIG_RT_GROUP_SCHED ++ init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); ++#endif ++ ++ for (j = 0; j < CPU_LOAD_IDX_MAX; j++) ++ rq->cpu_load[j] = 0; ++ ++ rq->last_load_update_tick = jiffies; ++ ++#ifdef CONFIG_SMP ++ rq->sd = NULL; ++ rq->rd = NULL; ++ rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; ++ rq->post_schedule = 0; ++ rq->active_balance = 0; ++ rq->next_balance = jiffies; ++ rq->push_cpu = 0; ++ rq->cpu = i; ++ rq->online = 0; ++ rq->idle_stamp = 0; ++ rq->avg_idle = 2*sysctl_sched_migration_cost; ++ rq->max_idle_balance_cost = sysctl_sched_migration_cost; ++ ++ INIT_LIST_HEAD(&rq->cfs_tasks); ++ ++ rq_attach_root(rq, &def_root_domain); ++#ifdef CONFIG_NO_HZ_COMMON ++ rq->nohz_flags = 0; ++#endif ++#ifdef CONFIG_NO_HZ_FULL ++ rq->last_sched_tick = 0; ++#endif ++#endif ++ init_rq_hrtick(rq); ++ atomic_set(&rq->nr_iowait, 0); ++ } ++ ++ set_load_weight(&init_task); ++ ++#ifdef CONFIG_PREEMPT_NOTIFIERS ++ INIT_HLIST_HEAD(&init_task.preempt_notifiers); ++#endif ++ ++ /* ++ * The boot idle thread does lazy MMU switching as well: ++ */ ++ atomic_inc(&init_mm.mm_count); ++ enter_lazy_tlb(&init_mm, current); ++ ++ /* ++ * During early bootup we pretend to be a normal task: ++ */ ++ current->sched_class = &fair_sched_class; ++ ++ /* ++ * Make us the idle thread. Technically, schedule() should not be ++ * called from this thread, however somewhere below it might be, ++ * but because we are the idle thread, we just pick up running again ++ * when this runqueue becomes "idle". ++ */ ++ init_idle(current, smp_processor_id()); ++ ++ calc_load_update = jiffies + LOAD_FREQ; ++ ++#ifdef CONFIG_SMP ++ zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT); ++ /* May be allocated at isolcpus cmdline parse time */ ++ if (cpu_isolated_map == NULL) ++ zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); ++ idle_thread_set_boot_cpu(); ++ set_cpu_rq_start_time(); ++#endif ++ init_sched_fair_class(); ++ ++ scheduler_running = 1; ++} ++ ++#ifdef CONFIG_DEBUG_ATOMIC_SLEEP ++static inline int preempt_count_equals(int preempt_offset) ++{ ++ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth(); ++ ++ return (nested == preempt_offset); ++} ++ ++void __might_sleep(const char *file, int line, int preempt_offset) ++{ ++ /* ++ * Blocking primitives will set (and therefore destroy) current->state, ++ * since we will exit with TASK_RUNNING make sure we enter with it, ++ * otherwise we will destroy state. ++ */ ++ WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change, ++ "do not call blocking ops when !TASK_RUNNING; " ++ "state=%lx set at [<%p>] %pS\n", ++ current->state, ++ (void *)current->task_state_change, ++ (void *)current->task_state_change); ++ ++ ___might_sleep(file, line, preempt_offset); ++} ++EXPORT_SYMBOL(__might_sleep); ++ ++void ___might_sleep(const char *file, int line, int preempt_offset) ++{ ++ static unsigned long prev_jiffy; /* ratelimiting */ ++ ++ rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */ ++ if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && ++ !is_idle_task(current)) || ++ system_state != SYSTEM_RUNNING || oops_in_progress) ++ return; ++ if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) ++ return; ++ prev_jiffy = jiffies; ++ ++ printk(KERN_ERR ++ "BUG: sleeping function called from invalid context at %s:%d\n", ++ file, line); ++ printk(KERN_ERR ++ "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", ++ in_atomic(), irqs_disabled(), ++ current->pid, current->comm); ++ ++ if (task_stack_end_corrupted(current)) ++ printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); ++ ++ debug_show_held_locks(current); ++ if (irqs_disabled()) ++ print_irqtrace_events(current); ++#ifdef CONFIG_DEBUG_PREEMPT ++ if (!preempt_count_equals(preempt_offset)) { ++ pr_err("Preemption disabled at:"); ++ print_ip_sym(current->preempt_disable_ip); ++ pr_cont("\n"); ++ } ++#endif ++ dump_stack(); ++} ++EXPORT_SYMBOL(___might_sleep); ++#endif ++ ++#ifdef CONFIG_MAGIC_SYSRQ ++static void normalize_task(struct rq *rq, struct task_struct *p) ++{ ++ const struct sched_class *prev_class = p->sched_class; ++ struct sched_attr attr = { ++ .sched_policy = SCHED_NORMAL, ++ }; ++ int old_prio = p->prio; ++ int queued; ++ ++ queued = task_on_rq_queued(p); ++ if (queued) ++ dequeue_task(rq, p, 0); ++ __setscheduler(rq, p, &attr, false); ++ if (queued) { ++ enqueue_task(rq, p, 0); ++ resched_curr(rq); ++ } ++ ++ check_class_changed(rq, p, prev_class, old_prio); ++} ++ ++void normalize_rt_tasks(void) ++{ ++ struct task_struct *g, *p; ++ unsigned long flags; ++ struct rq *rq; ++ ++ read_lock(&tasklist_lock); ++ for_each_process_thread(g, p) { ++ /* ++ * Only normalize user tasks: ++ */ ++ if (p->flags & PF_KTHREAD) ++ continue; ++ ++ p->se.exec_start = 0; ++#ifdef CONFIG_SCHEDSTATS ++ p->se.statistics.wait_start = 0; ++ p->se.statistics.sleep_start = 0; ++ p->se.statistics.block_start = 0; ++#endif ++ ++ if (!dl_task(p) && !rt_task(p)) { ++ /* ++ * Renice negative nice level userspace ++ * tasks back to 0: ++ */ ++ if (task_nice(p) < 0) ++ set_user_nice(p, 0); ++ continue; ++ } ++ ++ rq = task_rq_lock(p, &flags); ++ normalize_task(rq, p); ++ task_rq_unlock(rq, p, &flags); ++ } ++ read_unlock(&tasklist_lock); ++} ++ ++#endif /* CONFIG_MAGIC_SYSRQ */ ++ ++#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) ++/* ++ * These functions are only useful for the IA64 MCA handling, or kdb. ++ * ++ * They can only be called when the whole system has been ++ * stopped - every CPU needs to be quiescent, and no scheduling ++ * activity can take place. Using them for anything else would ++ * be a serious bug, and as a result, they aren't even visible ++ * under any other configuration. ++ */ ++ ++/** ++ * curr_task - return the current task for a given cpu. ++ * @cpu: the processor in question. ++ * ++ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! ++ * ++ * Return: The current task for @cpu. ++ */ ++struct task_struct *curr_task(int cpu) ++{ ++ return cpu_curr(cpu); ++} ++ ++#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ ++ ++#ifdef CONFIG_IA64 ++/** ++ * set_curr_task - set the current task for a given cpu. ++ * @cpu: the processor in question. ++ * @p: the task pointer to set. ++ * ++ * Description: This function must only be used when non-maskable interrupts ++ * are serviced on a separate stack. It allows the architecture to switch the ++ * notion of the current task on a cpu in a non-blocking manner. This function ++ * must be called with all CPU's synchronized, and interrupts disabled, the ++ * and caller must save the original value of the current task (see ++ * curr_task() above) and restore that value before reenabling interrupts and ++ * re-starting the system. ++ * ++ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! ++ */ ++void set_curr_task(int cpu, struct task_struct *p) ++{ ++ cpu_curr(cpu) = p; ++} ++ ++#endif ++ ++#ifdef CONFIG_CGROUP_SCHED ++/* task_group_lock serializes the addition/removal of task groups */ ++static DEFINE_SPINLOCK(task_group_lock); ++ ++static void free_sched_group(struct task_group *tg) ++{ ++ free_fair_sched_group(tg); ++ free_rt_sched_group(tg); ++ autogroup_free(tg); ++ kfree(tg); ++} ++ ++/* allocate runqueue etc for a new task group */ ++struct task_group *sched_create_group(struct task_group *parent) ++{ ++ struct task_group *tg; ++ ++ tg = kzalloc(sizeof(*tg), GFP_KERNEL); ++ if (!tg) ++ return ERR_PTR(-ENOMEM); ++ ++ if (!alloc_fair_sched_group(tg, parent)) ++ goto err; ++ ++ if (!alloc_rt_sched_group(tg, parent)) ++ goto err; ++ ++ return tg; ++ ++err: ++ free_sched_group(tg); ++ return ERR_PTR(-ENOMEM); ++} ++ ++void sched_online_group(struct task_group *tg, struct task_group *parent) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&task_group_lock, flags); ++ list_add_rcu(&tg->list, &task_groups); ++ ++ WARN_ON(!parent); /* root should already exist */ ++ ++ tg->parent = parent; ++ INIT_LIST_HEAD(&tg->children); ++ list_add_rcu(&tg->siblings, &parent->children); ++ spin_unlock_irqrestore(&task_group_lock, flags); ++} ++ ++/* rcu callback to free various structures associated with a task group */ ++static void free_sched_group_rcu(struct rcu_head *rhp) ++{ ++ /* now it should be safe to free those cfs_rqs */ ++ free_sched_group(container_of(rhp, struct task_group, rcu)); ++} ++ ++/* Destroy runqueue etc associated with a task group */ ++void sched_destroy_group(struct task_group *tg) ++{ ++ /* wait for possible concurrent references to cfs_rqs complete */ ++ call_rcu(&tg->rcu, free_sched_group_rcu); ++} ++ ++void sched_offline_group(struct task_group *tg) ++{ ++ unsigned long flags; ++ int i; ++ ++ /* end participation in shares distribution */ ++ for_each_possible_cpu(i) ++ unregister_fair_sched_group(tg, i); ++ ++ spin_lock_irqsave(&task_group_lock, flags); ++ list_del_rcu(&tg->list); ++ list_del_rcu(&tg->siblings); ++ spin_unlock_irqrestore(&task_group_lock, flags); ++} ++ ++/* change task's runqueue when it moves between groups. ++ * The caller of this function should have put the task in its new group ++ * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to ++ * reflect its new group. ++ */ ++void sched_move_task(struct task_struct *tsk) ++{ ++ struct task_group *tg; ++ int queued, running; ++ unsigned long flags; ++ struct rq *rq; ++ ++ rq = task_rq_lock(tsk, &flags); ++ ++ running = task_current(rq, tsk); ++ queued = task_on_rq_queued(tsk); ++ ++ if (queued) ++ dequeue_task(rq, tsk, 0); ++ if (unlikely(running)) ++ put_prev_task(rq, tsk); ++ ++ /* ++ * All callers are synchronized by task_rq_lock(); we do not use RCU ++ * which is pointless here. Thus, we pass "true" to task_css_check() ++ * to prevent lockdep warnings. ++ */ ++ tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), ++ struct task_group, css); ++ tg = autogroup_task_group(tsk, tg); ++ tsk->sched_task_group = tg; ++ ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ if (tsk->sched_class->task_move_group) ++ tsk->sched_class->task_move_group(tsk, queued); ++ else ++#endif ++ set_task_rq(tsk, task_cpu(tsk)); ++ ++ if (unlikely(running)) ++ tsk->sched_class->set_curr_task(rq); ++ if (queued) ++ enqueue_task(rq, tsk, 0); ++ ++ task_rq_unlock(rq, tsk, &flags); ++} ++#endif /* CONFIG_CGROUP_SCHED */ ++ ++#ifdef CONFIG_RT_GROUP_SCHED ++/* ++ * Ensure that the real time constraints are schedulable. ++ */ ++static DEFINE_MUTEX(rt_constraints_mutex); ++ ++/* Must be called with tasklist_lock held */ ++static inline int tg_has_rt_tasks(struct task_group *tg) ++{ ++ struct task_struct *g, *p; ++ ++ /* ++ * Autogroups do not have RT tasks; see autogroup_create(). ++ */ ++ if (task_group_is_autogroup(tg)) ++ return 0; ++ ++ for_each_process_thread(g, p) { ++ if (rt_task(p) && task_group(p) == tg) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++struct rt_schedulable_data { ++ struct task_group *tg; ++ u64 rt_period; ++ u64 rt_runtime; ++}; ++ ++static int tg_rt_schedulable(struct task_group *tg, void *data) ++{ ++ struct rt_schedulable_data *d = data; ++ struct task_group *child; ++ unsigned long total, sum = 0; ++ u64 period, runtime; ++ ++ period = ktime_to_ns(tg->rt_bandwidth.rt_period); ++ runtime = tg->rt_bandwidth.rt_runtime; ++ ++ if (tg == d->tg) { ++ period = d->rt_period; ++ runtime = d->rt_runtime; ++ } ++ ++ /* ++ * Cannot have more runtime than the period. ++ */ ++ if (runtime > period && runtime != RUNTIME_INF) ++ return -EINVAL; ++ ++ /* ++ * Ensure we don't starve existing RT tasks. ++ */ ++ if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg)) ++ return -EBUSY; ++ ++ total = to_ratio(period, runtime); ++ ++ /* ++ * Nobody can have more than the global setting allows. ++ */ ++ if (total > to_ratio(global_rt_period(), global_rt_runtime())) ++ return -EINVAL; ++ ++ /* ++ * The sum of our children's runtime should not exceed our own. ++ */ ++ list_for_each_entry_rcu(child, &tg->children, siblings) { ++ period = ktime_to_ns(child->rt_bandwidth.rt_period); ++ runtime = child->rt_bandwidth.rt_runtime; ++ ++ if (child == d->tg) { ++ period = d->rt_period; ++ runtime = d->rt_runtime; ++ } ++ ++ sum += to_ratio(period, runtime); ++ } ++ ++ if (sum > total) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) ++{ ++ int ret; ++ ++ struct rt_schedulable_data data = { ++ .tg = tg, ++ .rt_period = period, ++ .rt_runtime = runtime, ++ }; ++ ++ rcu_read_lock(); ++ ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data); ++ rcu_read_unlock(); ++ ++ return ret; ++} ++ ++static int tg_set_rt_bandwidth(struct task_group *tg, ++ u64 rt_period, u64 rt_runtime) ++{ ++ int i, err = 0; ++ ++ /* ++ * Disallowing the root group RT runtime is BAD, it would disallow the ++ * kernel creating (and or operating) RT threads. ++ */ ++ if (tg == &root_task_group && rt_runtime == 0) ++ return -EINVAL; ++ ++ /* No period doesn't make any sense. */ ++ if (rt_period == 0) ++ return -EINVAL; ++ ++ mutex_lock(&rt_constraints_mutex); ++ read_lock(&tasklist_lock); ++ err = __rt_schedulable(tg, rt_period, rt_runtime); ++ if (err) ++ goto unlock; ++ ++ raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); ++ tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); ++ tg->rt_bandwidth.rt_runtime = rt_runtime; ++ ++ for_each_possible_cpu(i) { ++ struct rt_rq *rt_rq = tg->rt_rq[i]; ++ ++ raw_spin_lock(&rt_rq->rt_runtime_lock); ++ rt_rq->rt_runtime = rt_runtime; ++ raw_spin_unlock(&rt_rq->rt_runtime_lock); ++ } ++ raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); ++unlock: ++ read_unlock(&tasklist_lock); ++ mutex_unlock(&rt_constraints_mutex); ++ ++ return err; ++} ++ ++static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us) ++{ ++ u64 rt_runtime, rt_period; ++ ++ rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period); ++ rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC; ++ if (rt_runtime_us < 0) ++ rt_runtime = RUNTIME_INF; ++ ++ return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); ++} ++ ++static long sched_group_rt_runtime(struct task_group *tg) ++{ ++ u64 rt_runtime_us; ++ ++ if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF) ++ return -1; ++ ++ rt_runtime_us = tg->rt_bandwidth.rt_runtime; ++ do_div(rt_runtime_us, NSEC_PER_USEC); ++ return rt_runtime_us; ++} ++ ++static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us) ++{ ++ u64 rt_runtime, rt_period; ++ ++ rt_period = (u64)rt_period_us * NSEC_PER_USEC; ++ rt_runtime = tg->rt_bandwidth.rt_runtime; ++ ++ return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); ++} ++ ++static long sched_group_rt_period(struct task_group *tg) ++{ ++ u64 rt_period_us; ++ ++ rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period); ++ do_div(rt_period_us, NSEC_PER_USEC); ++ return rt_period_us; ++} ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ ++#ifdef CONFIG_RT_GROUP_SCHED ++static int sched_rt_global_constraints(void) ++{ ++ int ret = 0; ++ ++ mutex_lock(&rt_constraints_mutex); ++ read_lock(&tasklist_lock); ++ ret = __rt_schedulable(NULL, 0, 0); ++ read_unlock(&tasklist_lock); ++ mutex_unlock(&rt_constraints_mutex); ++ ++ return ret; ++} ++ ++static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) ++{ ++ /* Don't accept realtime tasks when there is no way for them to run */ ++ if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0) ++ return 0; ++ ++ return 1; ++} ++ ++#else /* !CONFIG_RT_GROUP_SCHED */ ++static int sched_rt_global_constraints(void) ++{ ++ unsigned long flags; ++ int i, ret = 0; ++ ++ raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); ++ for_each_possible_cpu(i) { ++ struct rt_rq *rt_rq = &cpu_rq(i)->rt; ++ ++ raw_spin_lock(&rt_rq->rt_runtime_lock); ++ rt_rq->rt_runtime = global_rt_runtime(); ++ raw_spin_unlock(&rt_rq->rt_runtime_lock); ++ } ++ raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); ++ ++ return ret; ++} ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ ++static int sched_dl_global_validate(void) ++{ ++ u64 runtime = global_rt_runtime(); ++ u64 period = global_rt_period(); ++ u64 new_bw = to_ratio(period, runtime); ++ struct dl_bw *dl_b; ++ int cpu, ret = 0; ++ unsigned long flags; ++ ++ /* ++ * Here we want to check the bandwidth not being set to some ++ * value smaller than the currently allocated bandwidth in ++ * any of the root_domains. ++ * ++ * FIXME: Cycling on all the CPUs is overdoing, but simpler than ++ * cycling on root_domains... Discussion on different/better ++ * solutions is welcome! ++ */ ++ for_each_possible_cpu(cpu) { ++ rcu_read_lock_sched(); ++ dl_b = dl_bw_of(cpu); ++ ++ raw_spin_lock_irqsave(&dl_b->lock, flags); ++ if (new_bw < dl_b->total_bw) ++ ret = -EBUSY; ++ raw_spin_unlock_irqrestore(&dl_b->lock, flags); ++ ++ rcu_read_unlock_sched(); ++ ++ if (ret) ++ break; ++ } ++ ++ return ret; ++} ++ ++static void sched_dl_do_global(void) ++{ ++ u64 new_bw = -1; ++ struct dl_bw *dl_b; ++ int cpu; ++ unsigned long flags; ++ ++ def_dl_bandwidth.dl_period = global_rt_period(); ++ def_dl_bandwidth.dl_runtime = global_rt_runtime(); ++ ++ if (global_rt_runtime() != RUNTIME_INF) ++ new_bw = to_ratio(global_rt_period(), global_rt_runtime()); ++ ++ /* ++ * FIXME: As above... ++ */ ++ for_each_possible_cpu(cpu) { ++ rcu_read_lock_sched(); ++ dl_b = dl_bw_of(cpu); ++ ++ raw_spin_lock_irqsave(&dl_b->lock, flags); ++ dl_b->bw = new_bw; ++ raw_spin_unlock_irqrestore(&dl_b->lock, flags); ++ ++ rcu_read_unlock_sched(); ++ } ++} ++ ++static int sched_rt_global_validate(void) ++{ ++ if (sysctl_sched_rt_period <= 0) ++ return -EINVAL; ++ ++ if ((sysctl_sched_rt_runtime != RUNTIME_INF) && ++ (sysctl_sched_rt_runtime > sysctl_sched_rt_period)) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static void sched_rt_do_global(void) ++{ ++ def_rt_bandwidth.rt_runtime = global_rt_runtime(); ++ def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period()); ++} ++ ++int sched_rt_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, ++ loff_t *ppos) ++{ ++ int old_period, old_runtime; ++ static DEFINE_MUTEX(mutex); ++ int ret; ++ ++ mutex_lock(&mutex); ++ old_period = sysctl_sched_rt_period; ++ old_runtime = sysctl_sched_rt_runtime; ++ ++ ret = proc_dointvec(table, write, buffer, lenp, ppos); ++ ++ if (!ret && write) { ++ ret = sched_rt_global_validate(); ++ if (ret) ++ goto undo; ++ ++ ret = sched_dl_global_validate(); ++ if (ret) ++ goto undo; ++ ++ ret = sched_rt_global_constraints(); ++ if (ret) ++ goto undo; ++ ++ sched_rt_do_global(); ++ sched_dl_do_global(); ++ } ++ if (0) { ++undo: ++ sysctl_sched_rt_period = old_period; ++ sysctl_sched_rt_runtime = old_runtime; ++ } ++ mutex_unlock(&mutex); ++ ++ return ret; ++} ++ ++int sched_rr_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, ++ loff_t *ppos) ++{ ++ int ret; ++ static DEFINE_MUTEX(mutex); ++ ++ mutex_lock(&mutex); ++ ret = proc_dointvec(table, write, buffer, lenp, ppos); ++ /* make sure that internally we keep jiffies */ ++ /* also, writing zero resets timeslice to default */ ++ if (!ret && write) { ++ sched_rr_timeslice = sched_rr_timeslice <= 0 ? ++ RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice); ++ } ++ mutex_unlock(&mutex); ++ return ret; ++} ++ ++#ifdef CONFIG_CGROUP_SCHED ++ ++static inline struct task_group *css_tg(struct cgroup_subsys_state *css) ++{ ++ return css ? container_of(css, struct task_group, css) : NULL; ++} ++ ++static struct cgroup_subsys_state * ++cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) ++{ ++ struct task_group *parent = css_tg(parent_css); ++ struct task_group *tg; ++ ++ if (!parent) { ++ /* This is early initialization for the top cgroup */ ++ return &root_task_group.css; ++ } ++ ++ tg = sched_create_group(parent); ++ if (IS_ERR(tg)) ++ return ERR_PTR(-ENOMEM); ++ ++ return &tg->css; ++} ++ ++static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) ++{ ++ struct task_group *tg = css_tg(css); ++ struct task_group *parent = css_tg(css->parent); ++ ++ if (parent) ++ sched_online_group(tg, parent); ++ return 0; ++} ++ ++static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) ++{ ++ struct task_group *tg = css_tg(css); ++ ++ sched_destroy_group(tg); ++} ++ ++static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css) ++{ ++ struct task_group *tg = css_tg(css); ++ ++ sched_offline_group(tg); ++} ++ ++static void cpu_cgroup_fork(struct task_struct *task) ++{ ++ sched_move_task(task); ++} ++ ++static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css, ++ struct cgroup_taskset *tset) ++{ ++ struct task_struct *task; ++ ++ cgroup_taskset_for_each(task, tset) { ++#ifdef CONFIG_RT_GROUP_SCHED ++ if (!sched_rt_can_attach(css_tg(css), task)) ++ return -EINVAL; ++#else ++ /* We don't support RT-tasks being in separate groups */ ++ if (task->sched_class != &fair_sched_class) ++ return -EINVAL; ++#endif ++ } ++ return 0; ++} ++ ++static void cpu_cgroup_attach(struct cgroup_subsys_state *css, ++ struct cgroup_taskset *tset) ++{ ++ struct task_struct *task; ++ ++ cgroup_taskset_for_each(task, tset) ++ sched_move_task(task); ++} ++ ++static void cpu_cgroup_exit(struct cgroup_subsys_state *css, ++ struct cgroup_subsys_state *old_css, ++ struct task_struct *task) ++{ ++ /* ++ * cgroup_exit() is called in the copy_process() failure path. ++ * Ignore this case since the task hasn't ran yet, this avoids ++ * trying to poke a half freed task state from generic code. ++ */ ++ if (!(task->flags & PF_EXITING)) ++ return; ++ ++ sched_move_task(task); ++} ++ ++#ifdef CONFIG_FAIR_GROUP_SCHED ++static int cpu_shares_write_u64(struct cgroup_subsys_state *css, ++ struct cftype *cftype, u64 shareval) ++{ ++ return sched_group_set_shares(css_tg(css), scale_load(shareval)); ++} ++ ++static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ struct task_group *tg = css_tg(css); ++ ++ return (u64) scale_load_down(tg->shares); ++} ++ ++#ifdef CONFIG_CFS_BANDWIDTH ++static DEFINE_MUTEX(cfs_constraints_mutex); ++ ++const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ ++const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ ++ ++static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); ++ ++static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) ++{ ++ int i, ret = 0, runtime_enabled, runtime_was_enabled; ++ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; ++ ++ if (tg == &root_task_group) ++ return -EINVAL; ++ ++ /* ++ * Ensure we have at some amount of bandwidth every period. This is ++ * to prevent reaching a state of large arrears when throttled via ++ * entity_tick() resulting in prolonged exit starvation. ++ */ ++ if (quota < min_cfs_quota_period || period < min_cfs_quota_period) ++ return -EINVAL; ++ ++ /* ++ * Likewise, bound things on the otherside by preventing insane quota ++ * periods. This also allows us to normalize in computing quota ++ * feasibility. ++ */ ++ if (period > max_cfs_quota_period) ++ return -EINVAL; ++ ++ /* ++ * Prevent race between setting of cfs_rq->runtime_enabled and ++ * unthrottle_offline_cfs_rqs(). ++ */ ++ get_online_cpus(); ++ mutex_lock(&cfs_constraints_mutex); ++ ret = __cfs_schedulable(tg, period, quota); ++ if (ret) ++ goto out_unlock; ++ ++ runtime_enabled = quota != RUNTIME_INF; ++ runtime_was_enabled = cfs_b->quota != RUNTIME_INF; ++ /* ++ * If we need to toggle cfs_bandwidth_used, off->on must occur ++ * before making related changes, and on->off must occur afterwards ++ */ ++ if (runtime_enabled && !runtime_was_enabled) ++ cfs_bandwidth_usage_inc(); ++ raw_spin_lock_irq(&cfs_b->lock); ++ cfs_b->period = ns_to_ktime(period); ++ cfs_b->quota = quota; ++ ++ __refill_cfs_bandwidth_runtime(cfs_b); ++ /* restart the period timer (if active) to handle new period expiry */ ++ if (runtime_enabled && cfs_b->timer_active) { ++ /* force a reprogram */ ++ __start_cfs_bandwidth(cfs_b, true); ++ } ++ raw_spin_unlock_irq(&cfs_b->lock); ++ ++ for_each_online_cpu(i) { ++ struct cfs_rq *cfs_rq = tg->cfs_rq[i]; ++ struct rq *rq = cfs_rq->rq; ++ ++ raw_spin_lock_irq(&rq->lock); ++ cfs_rq->runtime_enabled = runtime_enabled; ++ cfs_rq->runtime_remaining = 0; ++ ++ if (cfs_rq->throttled) ++ unthrottle_cfs_rq(cfs_rq); ++ raw_spin_unlock_irq(&rq->lock); ++ } ++ if (runtime_was_enabled && !runtime_enabled) ++ cfs_bandwidth_usage_dec(); ++out_unlock: ++ mutex_unlock(&cfs_constraints_mutex); ++ put_online_cpus(); ++ ++ return ret; ++} ++ ++int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) ++{ ++ u64 quota, period; ++ ++ period = ktime_to_ns(tg->cfs_bandwidth.period); ++ if (cfs_quota_us < 0) ++ quota = RUNTIME_INF; ++ else ++ quota = (u64)cfs_quota_us * NSEC_PER_USEC; ++ ++ return tg_set_cfs_bandwidth(tg, period, quota); ++} ++ ++long tg_get_cfs_quota(struct task_group *tg) ++{ ++ u64 quota_us; ++ ++ if (tg->cfs_bandwidth.quota == RUNTIME_INF) ++ return -1; ++ ++ quota_us = tg->cfs_bandwidth.quota; ++ do_div(quota_us, NSEC_PER_USEC); ++ ++ return quota_us; ++} ++ ++int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) ++{ ++ u64 quota, period; ++ ++ period = (u64)cfs_period_us * NSEC_PER_USEC; ++ quota = tg->cfs_bandwidth.quota; ++ ++ return tg_set_cfs_bandwidth(tg, period, quota); ++} ++ ++long tg_get_cfs_period(struct task_group *tg) ++{ ++ u64 cfs_period_us; ++ ++ cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); ++ do_div(cfs_period_us, NSEC_PER_USEC); ++ ++ return cfs_period_us; ++} ++ ++static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return tg_get_cfs_quota(css_tg(css)); ++} ++ ++static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, ++ struct cftype *cftype, s64 cfs_quota_us) ++{ ++ return tg_set_cfs_quota(css_tg(css), cfs_quota_us); ++} ++ ++static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return tg_get_cfs_period(css_tg(css)); ++} ++ ++static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, ++ struct cftype *cftype, u64 cfs_period_us) ++{ ++ return tg_set_cfs_period(css_tg(css), cfs_period_us); ++} ++ ++struct cfs_schedulable_data { ++ struct task_group *tg; ++ u64 period, quota; ++}; ++ ++/* ++ * normalize group quota/period to be quota/max_period ++ * note: units are usecs ++ */ ++static u64 normalize_cfs_quota(struct task_group *tg, ++ struct cfs_schedulable_data *d) ++{ ++ u64 quota, period; ++ ++ if (tg == d->tg) { ++ period = d->period; ++ quota = d->quota; ++ } else { ++ period = tg_get_cfs_period(tg); ++ quota = tg_get_cfs_quota(tg); ++ } ++ ++ /* note: these should typically be equivalent */ ++ if (quota == RUNTIME_INF || quota == -1) ++ return RUNTIME_INF; ++ ++ return to_ratio(period, quota); ++} ++ ++static int tg_cfs_schedulable_down(struct task_group *tg, void *data) ++{ ++ struct cfs_schedulable_data *d = data; ++ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; ++ s64 quota = 0, parent_quota = -1; ++ ++ if (!tg->parent) { ++ quota = RUNTIME_INF; ++ } else { ++ struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; ++ ++ quota = normalize_cfs_quota(tg, d); ++ parent_quota = parent_b->hierarchical_quota; ++ ++ /* ++ * ensure max(child_quota) <= parent_quota, inherit when no ++ * limit is set ++ */ ++ if (quota == RUNTIME_INF) ++ quota = parent_quota; ++ else if (parent_quota != RUNTIME_INF && quota > parent_quota) ++ return -EINVAL; ++ } ++ cfs_b->hierarchical_quota = quota; ++ ++ return 0; ++} ++ ++static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) ++{ ++ int ret; ++ struct cfs_schedulable_data data = { ++ .tg = tg, ++ .period = period, ++ .quota = quota, ++ }; ++ ++ if (quota != RUNTIME_INF) { ++ do_div(data.period, NSEC_PER_USEC); ++ do_div(data.quota, NSEC_PER_USEC); ++ } ++ ++ rcu_read_lock(); ++ ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); ++ rcu_read_unlock(); ++ ++ return ret; ++} ++ ++static int cpu_stats_show(struct seq_file *sf, void *v) ++{ ++ struct task_group *tg = css_tg(seq_css(sf)); ++ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; ++ ++ seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); ++ seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); ++ seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); ++ ++ return 0; ++} ++#endif /* CONFIG_CFS_BANDWIDTH */ ++#endif /* CONFIG_FAIR_GROUP_SCHED */ ++ ++#ifdef CONFIG_RT_GROUP_SCHED ++static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, ++ struct cftype *cft, s64 val) ++{ ++ return sched_group_set_rt_runtime(css_tg(css), val); ++} ++ ++static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return sched_group_rt_runtime(css_tg(css)); ++} ++ ++static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, ++ struct cftype *cftype, u64 rt_period_us) ++{ ++ return sched_group_set_rt_period(css_tg(css), rt_period_us); ++} ++ ++static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return sched_group_rt_period(css_tg(css)); ++} ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ ++static struct cftype cpu_files[] = { ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ { ++ .name = "shares", ++ .read_u64 = cpu_shares_read_u64, ++ .write_u64 = cpu_shares_write_u64, ++ }, ++#endif ++#ifdef CONFIG_CFS_BANDWIDTH ++ { ++ .name = "cfs_quota_us", ++ .read_s64 = cpu_cfs_quota_read_s64, ++ .write_s64 = cpu_cfs_quota_write_s64, ++ }, ++ { ++ .name = "cfs_period_us", ++ .read_u64 = cpu_cfs_period_read_u64, ++ .write_u64 = cpu_cfs_period_write_u64, ++ }, ++ { ++ .name = "stat", ++ .seq_show = cpu_stats_show, ++ }, ++#endif ++#ifdef CONFIG_RT_GROUP_SCHED ++ { ++ .name = "rt_runtime_us", ++ .read_s64 = cpu_rt_runtime_read, ++ .write_s64 = cpu_rt_runtime_write, ++ }, ++ { ++ .name = "rt_period_us", ++ .read_u64 = cpu_rt_period_read_uint, ++ .write_u64 = cpu_rt_period_write_uint, ++ }, ++#endif ++ { } /* terminate */ ++}; ++ ++struct cgroup_subsys cpu_cgrp_subsys = { ++ .css_alloc = cpu_cgroup_css_alloc, ++ .css_free = cpu_cgroup_css_free, ++ .css_online = cpu_cgroup_css_online, ++ .css_offline = cpu_cgroup_css_offline, ++ .fork = cpu_cgroup_fork, ++ .can_attach = cpu_cgroup_can_attach, ++ .attach = cpu_cgroup_attach, ++ .exit = cpu_cgroup_exit, ++ .legacy_cftypes = cpu_files, ++ .early_init = 1, ++}; ++ ++#endif /* CONFIG_CGROUP_SCHED */ ++ ++void dump_cpu_task(int cpu) ++{ ++ pr_info("Task dump for CPU %d:\n", cpu); ++ sched_show_task(cpu_curr(cpu)); ++} +diff -Nur linux-4.1.10.orig/kernel/sched/cputime.c linux-4.1.10/kernel/sched/cputime.c +--- linux-4.1.10.orig/kernel/sched/cputime.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/cputime.c 2015-10-07 18:00:08.000000000 +0200 @@ -675,37 +675,45 @@ void vtime_account_system(struct task_struct *tsk) @@ -18528,9 +42798,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/cputime.c linux-4.1.6/kernel/sched/cputi } -diff -Nur linux-4.1.6.orig/kernel/sched/deadline.c linux-4.1.6/kernel/sched/deadline.c ---- linux-4.1.6.orig/kernel/sched/deadline.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/deadline.c 2015-09-08 23:49:08.109866942 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/deadline.c linux-4.1.10/kernel/sched/deadline.c +--- linux-4.1.10.orig/kernel/sched/deadline.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/deadline.c 2015-10-07 18:00:08.000000000 +0200 @@ -637,6 +637,7 @@ hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); @@ -18539,9 +42809,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/deadline.c linux-4.1.6/kernel/sched/dead } static -diff -Nur linux-4.1.6.orig/kernel/sched/debug.c linux-4.1.6/kernel/sched/debug.c ---- linux-4.1.6.orig/kernel/sched/debug.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/debug.c 2015-09-08 23:49:08.109866942 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/debug.c linux-4.1.10/kernel/sched/debug.c +--- linux-4.1.10.orig/kernel/sched/debug.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/debug.c 2015-10-07 18:00:08.000000000 +0200 @@ -260,6 +260,9 @@ P(rt_throttled); PN(rt_time); @@ -18563,9 +42833,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/debug.c linux-4.1.6/kernel/sched/debug.c #undef PN #undef __PN #undef P -diff -Nur linux-4.1.6.orig/kernel/sched/fair.c linux-4.1.6/kernel/sched/fair.c ---- linux-4.1.6.orig/kernel/sched/fair.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/fair.c 2015-09-08 23:49:08.113866499 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/fair.c linux-4.1.10/kernel/sched/fair.c +--- linux-4.1.10.orig/kernel/sched/fair.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/fair.c 2015-10-07 18:00:08.000000000 +0200 @@ -3201,7 +3201,7 @@ ideal_runtime = sched_slice(cfs_rq, curr); delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; @@ -18638,9 +42908,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/fair.c linux-4.1.6/kernel/sched/fair.c } else check_preempt_curr(rq, p, 0); } -diff -Nur linux-4.1.6.orig/kernel/sched/features.h linux-4.1.6/kernel/sched/features.h ---- linux-4.1.6.orig/kernel/sched/features.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/features.h 2015-09-08 23:49:08.113866499 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/features.h linux-4.1.10/kernel/sched/features.h +--- linux-4.1.10.orig/kernel/sched/features.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/features.h 2015-10-07 18:00:08.000000000 +0200 @@ -50,11 +50,19 @@ */ SCHED_FEAT(NONTASK_CAPACITY, true) @@ -18661,9 +42931,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/features.h linux-4.1.6/kernel/sched/feat #ifdef HAVE_RT_PUSH_IPI /* -diff -Nur linux-4.1.6.orig/kernel/sched/Makefile linux-4.1.6/kernel/sched/Makefile ---- linux-4.1.6.orig/kernel/sched/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/Makefile 2015-09-08 23:49:08.105867384 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/Makefile linux-4.1.10/kernel/sched/Makefile +--- linux-4.1.10.orig/kernel/sched/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/Makefile 2015-10-07 18:00:08.000000000 +0200 @@ -13,7 +13,7 @@ obj-y += core.o proc.o clock.o cputime.o @@ -18673,9 +42943,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/Makefile linux-4.1.6/kernel/sched/Makefi obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o -diff -Nur linux-4.1.6.orig/kernel/sched/rt.c linux-4.1.6/kernel/sched/rt.c ---- linux-4.1.6.orig/kernel/sched/rt.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/rt.c 2015-09-08 23:49:08.113866499 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/rt.c linux-4.1.10/kernel/sched/rt.c +--- linux-4.1.10.orig/kernel/sched/rt.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/rt.c 2015-10-07 18:00:08.000000000 +0200 @@ -44,6 +44,7 @@ hrtimer_init(&rt_b->rt_period_timer, @@ -18692,9 +42962,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/rt.c linux-4.1.6/kernel/sched/rt.c #endif #endif /* CONFIG_SMP */ /* We start is dequeued state, because no RT tasks are queued */ -diff -Nur linux-4.1.6.orig/kernel/sched/sched.h linux-4.1.6/kernel/sched/sched.h ---- linux-4.1.6.orig/kernel/sched/sched.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/sched.h 2015-09-08 23:49:08.113866499 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/sched.h linux-4.1.10/kernel/sched/sched.h +--- linux-4.1.10.orig/kernel/sched/sched.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/sched.h 2015-10-07 18:00:08.000000000 +0200 @@ -1092,6 +1092,7 @@ #define WF_SYNC 0x01 /* waker goes to sleep after wakeup */ #define WF_FORK 0x02 /* child wakeup after fork */ @@ -18719,9 +42989,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/sched.h linux-4.1.6/kernel/sched/sched.h extern struct rt_bandwidth def_rt_bandwidth; extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); -diff -Nur linux-4.1.6.orig/kernel/sched/wait-simple.c linux-4.1.6/kernel/sched/wait-simple.c ---- linux-4.1.6.orig/kernel/sched/wait-simple.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/kernel/sched/wait-simple.c 2015-09-08 23:49:08.113866499 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/wait-simple.c linux-4.1.10/kernel/sched/wait-simple.c +--- linux-4.1.10.orig/kernel/sched/wait-simple.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/sched/wait-simple.c 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,115 @@ +/* + * Simple waitqueues without fancy flags and callbacks @@ -18838,9 +43108,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/wait-simple.c linux-4.1.6/kernel/sched/w + return woken; +} +EXPORT_SYMBOL(__swait_wake); -diff -Nur linux-4.1.6.orig/kernel/sched/work-simple.c linux-4.1.6/kernel/sched/work-simple.c ---- linux-4.1.6.orig/kernel/sched/work-simple.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/kernel/sched/work-simple.c 2015-09-08 23:49:08.113866499 +0200 +diff -Nur linux-4.1.10.orig/kernel/sched/work-simple.c linux-4.1.10/kernel/sched/work-simple.c +--- linux-4.1.10.orig/kernel/sched/work-simple.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/sched/work-simple.c 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2014 BMW Car IT GmbH, Daniel Wagner daniel.wagner@bmw-carit.de @@ -19014,9 +43284,9 @@ diff -Nur linux-4.1.6.orig/kernel/sched/work-simple.c linux-4.1.6/kernel/sched/w + mutex_unlock(&worker_mutex); +} +EXPORT_SYMBOL_GPL(swork_put); -diff -Nur linux-4.1.6.orig/kernel/signal.c linux-4.1.6/kernel/signal.c ---- linux-4.1.6.orig/kernel/signal.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/signal.c 2015-09-08 23:49:08.113866499 +0200 +diff -Nur linux-4.1.10.orig/kernel/signal.c linux-4.1.10/kernel/signal.c +--- linux-4.1.10.orig/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/signal.c 2015-10-07 18:00:08.000000000 +0200 @@ -14,6 +14,7 @@ #include <linux/export.h> #include <linux/init.h> @@ -19261,9 +43531,9 @@ diff -Nur linux-4.1.6.orig/kernel/signal.c linux-4.1.6/kernel/signal.c freezable_schedule(); } else { /* -diff -Nur linux-4.1.6.orig/kernel/softirq.c linux-4.1.6/kernel/softirq.c ---- linux-4.1.6.orig/kernel/softirq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/softirq.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/softirq.c linux-4.1.10/kernel/softirq.c +--- linux-4.1.10.orig/kernel/softirq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/softirq.c 2015-10-07 18:00:08.000000000 +0200 @@ -21,10 +21,12 @@ #include <linux/freezer.h> #include <linux/kthread.h> @@ -20150,9 +44420,9 @@ diff -Nur linux-4.1.6.orig/kernel/softirq.c linux-4.1.6/kernel/softirq.c .thread_should_run = ksoftirqd_should_run, .thread_fn = run_ksoftirqd, .thread_comm = "ksoftirqd/%u", -diff -Nur linux-4.1.6.orig/kernel/stop_machine.c linux-4.1.6/kernel/stop_machine.c ---- linux-4.1.6.orig/kernel/stop_machine.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/stop_machine.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/stop_machine.c linux-4.1.10/kernel/stop_machine.c +--- linux-4.1.10.orig/kernel/stop_machine.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/stop_machine.c 2015-10-07 18:00:08.000000000 +0200 @@ -30,12 +30,12 @@ atomic_t nr_todo; /* nr left to execute */ bool executed; /* actually executed? */ @@ -20413,9 +44683,9 @@ diff -Nur linux-4.1.6.orig/kernel/stop_machine.c linux-4.1.6/kernel/stop_machine cpu_relax(); mutex_unlock(&stop_cpus_mutex); -diff -Nur linux-4.1.6.orig/kernel/time/hrtimer.c linux-4.1.6/kernel/time/hrtimer.c ---- linux-4.1.6.orig/kernel/time/hrtimer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/hrtimer.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/hrtimer.c linux-4.1.10/kernel/time/hrtimer.c +--- linux-4.1.10.orig/kernel/time/hrtimer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/hrtimer.c 2015-10-07 18:00:08.000000000 +0200 @@ -48,11 +48,13 @@ #include <linux/sched/rt.h> #include <linux/sched/deadline.h> @@ -21020,9 +45290,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/hrtimer.c linux-4.1.6/kernel/time/hrtimer } /** -diff -Nur linux-4.1.6.orig/kernel/time/itimer.c linux-4.1.6/kernel/time/itimer.c ---- linux-4.1.6.orig/kernel/time/itimer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/itimer.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/itimer.c linux-4.1.10/kernel/time/itimer.c +--- linux-4.1.10.orig/kernel/time/itimer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/itimer.c 2015-10-07 18:00:08.000000000 +0200 @@ -213,6 +213,7 @@ /* We are sharing ->siglock with it_real_fn() */ if (hrtimer_try_to_cancel(timer) < 0) { @@ -21031,9 +45301,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/itimer.c linux-4.1.6/kernel/time/itimer.c goto again; } expires = timeval_to_ktime(value->it_value); -diff -Nur linux-4.1.6.orig/kernel/time/jiffies.c linux-4.1.6/kernel/time/jiffies.c ---- linux-4.1.6.orig/kernel/time/jiffies.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/jiffies.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/jiffies.c linux-4.1.10/kernel/time/jiffies.c +--- linux-4.1.10.orig/kernel/time/jiffies.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/jiffies.c 2015-10-07 18:00:08.000000000 +0200 @@ -74,7 +74,8 @@ .max_cycles = 10, }; @@ -21056,9 +45326,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/jiffies.c linux-4.1.6/kernel/time/jiffies return ret; } EXPORT_SYMBOL(get_jiffies_64); -diff -Nur linux-4.1.6.orig/kernel/time/ntp.c linux-4.1.6/kernel/time/ntp.c ---- linux-4.1.6.orig/kernel/time/ntp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/ntp.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/ntp.c linux-4.1.10/kernel/time/ntp.c +--- linux-4.1.10.orig/kernel/time/ntp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/ntp.c 2015-10-07 18:00:08.000000000 +0200 @@ -10,6 +10,7 @@ #include <linux/workqueue.h> #include <linux/hrtimer.h> @@ -21120,9 +45390,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/ntp.c linux-4.1.6/kernel/time/ntp.c #else void ntp_notify_cmos_timer(void) { } -diff -Nur linux-4.1.6.orig/kernel/time/posix-cpu-timers.c linux-4.1.6/kernel/time/posix-cpu-timers.c ---- linux-4.1.6.orig/kernel/time/posix-cpu-timers.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/posix-cpu-timers.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/posix-cpu-timers.c linux-4.1.10/kernel/time/posix-cpu-timers.c +--- linux-4.1.10.orig/kernel/time/posix-cpu-timers.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/posix-cpu-timers.c 2015-10-07 18:00:08.000000000 +0200 @@ -3,6 +3,7 @@ */ @@ -21370,9 +45640,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/posix-cpu-timers.c linux-4.1.6/kernel/tim /* * Set one of the process-wide special case CPU timers or RLIMIT_CPU. * The tsk->sighand->siglock must be held by the caller. -diff -Nur linux-4.1.6.orig/kernel/time/posix-timers.c linux-4.1.6/kernel/time/posix-timers.c ---- linux-4.1.6.orig/kernel/time/posix-timers.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/posix-timers.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/posix-timers.c linux-4.1.10/kernel/time/posix-timers.c +--- linux-4.1.10.orig/kernel/time/posix-timers.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/posix-timers.c 2015-10-07 18:00:08.000000000 +0200 @@ -499,6 +499,7 @@ static struct pid *good_sigevent(sigevent_t * event) { @@ -21468,9 +45738,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/posix-timers.c linux-4.1.6/kernel/time/po goto retry_delete; } list_del(&timer->list); -diff -Nur linux-4.1.6.orig/kernel/time/tick-common.c linux-4.1.6/kernel/time/tick-common.c ---- linux-4.1.6.orig/kernel/time/tick-common.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/tick-common.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/tick-common.c linux-4.1.10/kernel/time/tick-common.c +--- linux-4.1.10.orig/kernel/time/tick-common.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/tick-common.c 2015-10-07 18:00:08.000000000 +0200 @@ -78,13 +78,15 @@ static void tick_periodic(int cpu) { @@ -21501,9 +45771,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/tick-common.c linux-4.1.6/kernel/time/tic clockevents_set_state(dev, CLOCK_EVT_STATE_ONESHOT); -diff -Nur linux-4.1.6.orig/kernel/time/tick-sched.c linux-4.1.6/kernel/time/tick-sched.c ---- linux-4.1.6.orig/kernel/time/tick-sched.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/tick-sched.c 2015-09-08 23:49:08.117866054 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/tick-sched.c linux-4.1.10/kernel/time/tick-sched.c +--- linux-4.1.10.orig/kernel/time/tick-sched.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/tick-sched.c 2015-10-07 18:00:08.000000000 +0200 @@ -62,7 +62,8 @@ return; @@ -21603,9 +45873,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/tick-sched.c linux-4.1.6/kernel/time/tick ts->sched_timer.function = tick_sched_timer; /* Get the next period (per cpu) */ -diff -Nur linux-4.1.6.orig/kernel/time/timekeeping.c linux-4.1.6/kernel/time/timekeeping.c ---- linux-4.1.6.orig/kernel/time/timekeeping.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/timekeeping.c 2015-09-08 23:49:08.125865168 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.c linux-4.1.10/kernel/time/timekeeping.c +--- linux-4.1.10.orig/kernel/time/timekeeping.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/timekeeping.c 2015-10-07 18:00:08.000000000 +0200 @@ -2065,8 +2065,10 @@ */ void xtime_update(unsigned long ticks) @@ -21619,9 +45889,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/timekeeping.c linux-4.1.6/kernel/time/tim + raw_spin_unlock(&jiffies_lock); update_wall_time(); } -diff -Nur linux-4.1.6.orig/kernel/time/timekeeping.h linux-4.1.6/kernel/time/timekeeping.h ---- linux-4.1.6.orig/kernel/time/timekeeping.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/timekeeping.h 2015-09-08 23:49:08.125865168 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.h linux-4.1.10/kernel/time/timekeeping.h +--- linux-4.1.10.orig/kernel/time/timekeeping.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/timekeeping.h 2015-10-07 18:00:08.000000000 +0200 @@ -22,7 +22,8 @@ extern void do_timer(unsigned long ticks); extern void update_wall_time(void); @@ -21632,9 +45902,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/timekeeping.h linux-4.1.6/kernel/time/tim #define CS_NAME_LEN 32 -diff -Nur linux-4.1.6.orig/kernel/time/timer.c linux-4.1.6/kernel/time/timer.c ---- linux-4.1.6.orig/kernel/time/timer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/timer.c 2015-09-08 23:49:08.125865168 +0200 +diff -Nur linux-4.1.10.orig/kernel/time/timer.c linux-4.1.10/kernel/time/timer.c +--- linux-4.1.10.orig/kernel/time/timer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/timer.c 2015-10-07 18:00:08.000000000 +0200 @@ -78,6 +78,9 @@ struct tvec_base { spinlock_t lock; @@ -21836,9 +46106,9 @@ diff -Nur linux-4.1.6.orig/kernel/time/timer.c linux-4.1.6/kernel/time/timer.c for (j = 0; j < TVN_SIZE; j++) { INIT_LIST_HEAD(base->tv5.vec + j); -diff -Nur linux-4.1.6.orig/kernel/trace/Kconfig linux-4.1.6/kernel/trace/Kconfig ---- linux-4.1.6.orig/kernel/trace/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/Kconfig 2015-09-08 23:49:08.125865168 +0200 +diff -Nur linux-4.1.10.orig/kernel/trace/Kconfig linux-4.1.10/kernel/trace/Kconfig +--- linux-4.1.10.orig/kernel/trace/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/Kconfig 2015-10-07 18:00:08.000000000 +0200 @@ -187,6 +187,24 @@ enabled. This option and the preempt-off timing option can be used together or separately.) @@ -21964,9 +46234,9 @@ diff -Nur linux-4.1.6.orig/kernel/trace/Kconfig linux-4.1.6/kernel/trace/Kconfig config ENABLE_DEFAULT_TRACERS bool "Trace process context switches and events" depends on !GENERIC_TRACER -diff -Nur linux-4.1.6.orig/kernel/trace/latency_hist.c linux-4.1.6/kernel/trace/latency_hist.c ---- linux-4.1.6.orig/kernel/trace/latency_hist.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/kernel/trace/latency_hist.c 2015-09-08 23:49:08.125865168 +0200 +diff -Nur linux-4.1.10.orig/kernel/trace/latency_hist.c linux-4.1.10/kernel/trace/latency_hist.c +--- linux-4.1.10.orig/kernel/trace/latency_hist.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/trace/latency_hist.c 2015-10-07 18:00:08.000000000 +0200 @@ -0,0 +1,1178 @@ +/* + * kernel/trace/latency_hist.c @@ -23146,9 +47416,9 @@ diff -Nur linux-4.1.6.orig/kernel/trace/latency_hist.c linux-4.1.6/kernel/trace/ +} + +device_initcall(latency_hist_init); -diff -Nur linux-4.1.6.orig/kernel/trace/Makefile linux-4.1.6/kernel/trace/Makefile ---- linux-4.1.6.orig/kernel/trace/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/Makefile 2015-09-08 23:49:08.125865168 +0200 +diff -Nur linux-4.1.10.orig/kernel/trace/Makefile linux-4.1.10/kernel/trace/Makefile +--- linux-4.1.10.orig/kernel/trace/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/Makefile 2015-10-07 18:00:08.000000000 +0200 @@ -36,6 +36,10 @@ obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o @@ -23160,9 +47430,9 @@ diff -Nur linux-4.1.6.orig/kernel/trace/Makefile linux-4.1.6/kernel/trace/Makefi obj-$(CONFIG_NOP_TRACER) += trace_nop.o obj-$(CONFIG_STACK_TRACER) += trace_stack.o obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o -diff -Nur linux-4.1.6.orig/kernel/trace/trace.c linux-4.1.6/kernel/trace/trace.c ---- linux-4.1.6.orig/kernel/trace/trace.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace.c 2015-09-08 23:49:08.125865168 +0200 +diff -Nur linux-4.1.10.orig/kernel/trace/trace.c linux-4.1.10/kernel/trace/trace.c +--- linux-4.1.10.orig/kernel/trace/trace.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace.c 2015-10-07 18:00:08.000000000 +0200 @@ -1630,6 +1630,7 @@ struct task_struct *tsk = current; @@ -23230,9 +47500,9 @@ diff -Nur linux-4.1.6.orig/kernel/trace/trace.c linux-4.1.6/kernel/trace/trace.c } void -diff -Nur linux-4.1.6.orig/kernel/trace/trace_events.c linux-4.1.6/kernel/trace/trace_events.c ---- linux-4.1.6.orig/kernel/trace/trace_events.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace_events.c 2015-09-08 23:49:08.125865168 +0200 +diff -Nur linux-4.1.10.orig/kernel/trace/trace_events.c linux-4.1.10/kernel/trace/trace_events.c +--- linux-4.1.10.orig/kernel/trace/trace_events.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace_events.c 2015-10-07 18:00:08.000000000 +0200 @@ -162,6 +162,8 @@ __common_field(unsigned char, flags); __common_field(unsigned char, preempt_count); @@ -23242,9 +47512,9 @@ diff -Nur linux-4.1.6.orig/kernel/trace/trace_events.c linux-4.1.6/kernel/trace/ return ret; } -diff -Nur linux-4.1.6.orig/kernel/trace/trace.h linux-4.1.6/kernel/trace/trace.h ---- linux-4.1.6.orig/kernel/trace/trace.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace.h 2015-09-08 23:49:08.125865168 +0200 +diff -Nur linux-4.1.10.orig/kernel/trace/trace.h linux-4.1.10/kernel/trace/trace.h +--- linux-4.1.10.orig/kernel/trace/trace.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace.h 2015-10-07 18:00:08.000000000 +0200 @@ -120,6 +120,7 @@ * NEED_RESCHED - reschedule is requested * HARDIRQ - inside an interrupt handler @@ -23261,9 +47531,9 @@ diff -Nur linux-4.1.6.orig/kernel/trace/trace.h linux-4.1.6/kernel/trace/trace.h }; #define TRACE_BUF_SIZE 1024 -diff -Nur linux-4.1.6.orig/kernel/trace/trace_irqsoff.c linux-4.1.6/kernel/trace/trace_irqsoff.c ---- linux-4.1.6.orig/kernel/trace/trace_irqsoff.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace_irqsoff.c 2015-09-08 23:49:08.129864725 +0200 +diff -Nur linux-4.1.10.orig/kernel/trace/trace_irqsoff.c linux-4.1.10/kernel/trace/trace_irqsoff.c +--- linux-4.1.10.orig/kernel/trace/trace_irqsoff.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace_irqsoff.c 2015-10-07 18:00:08.000000000 +0200 @@ -13,6 +13,7 @@ #include <linux/uaccess.h> #include <linux/module.h> @@ -23290,7 +47560,7 @@ diff -Nur linux-4.1.6.orig/kernel/trace/trace_irqsoff.c linux-4.1.6/kernel/trace #ifdef CONFIG_PROVE_LOCKING void time_hardirqs_on(unsigned long a0, unsigned long a1) { -+ trace_preemptirqsoff_hist(IRQS_ON, 0); ++ trace_preemptirqsoff_hist_rcuidle(IRQS_ON, 0); if (!preempt_trace() && irq_trace()) stop_critical_timing(a0, a1); } @@ -23298,7 +47568,7 @@ diff -Nur linux-4.1.6.orig/kernel/trace/trace_irqsoff.c linux-4.1.6/kernel/trace { if (!preempt_trace() && irq_trace()) start_critical_timing(a0, a1); -+ trace_preemptirqsoff_hist(IRQS_OFF, 1); ++ trace_preemptirqsoff_hist_rcuidle(IRQS_OFF, 1); } #else /* !CONFIG_PROVE_LOCKING */ @@ -23347,9 +47617,9 @@ diff -Nur linux-4.1.6.orig/kernel/trace/trace_irqsoff.c linux-4.1.6/kernel/trace if (preempt_trace() && !irq_trace()) start_critical_timing(a0, a1); } -diff -Nur linux-4.1.6.orig/kernel/trace/trace_output.c linux-4.1.6/kernel/trace/trace_output.c ---- linux-4.1.6.orig/kernel/trace/trace_output.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace_output.c 2015-09-08 23:49:08.137863836 +0200 +diff -Nur linux-4.1.10.orig/kernel/trace/trace_output.c linux-4.1.10/kernel/trace/trace_output.c +--- linux-4.1.10.orig/kernel/trace/trace_output.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace_output.c 2015-10-07 18:00:08.000000000 +0200 @@ -430,6 +430,7 @@ { char hardsoft_irq; @@ -23395,9 +47665,9 @@ diff -Nur linux-4.1.6.orig/kernel/trace/trace_output.c linux-4.1.6/kernel/trace/ return !trace_seq_has_overflowed(s); } -diff -Nur linux-4.1.6.orig/kernel/user.c linux-4.1.6/kernel/user.c ---- linux-4.1.6.orig/kernel/user.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/user.c 2015-09-08 23:49:08.137863836 +0200 +diff -Nur linux-4.1.10.orig/kernel/user.c linux-4.1.10/kernel/user.c +--- linux-4.1.10.orig/kernel/user.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/user.c 2015-10-07 18:00:08.000000000 +0200 @@ -161,11 +161,11 @@ if (!up) return; @@ -23412,9 +47682,9 @@ diff -Nur linux-4.1.6.orig/kernel/user.c linux-4.1.6/kernel/user.c } struct user_struct *alloc_uid(kuid_t uid) -diff -Nur linux-4.1.6.orig/kernel/watchdog.c linux-4.1.6/kernel/watchdog.c ---- linux-4.1.6.orig/kernel/watchdog.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/watchdog.c 2015-09-08 23:49:08.421832347 +0200 +diff -Nur linux-4.1.10.orig/kernel/watchdog.c linux-4.1.10/kernel/watchdog.c +--- linux-4.1.10.orig/kernel/watchdog.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/watchdog.c 2015-10-07 18:00:08.000000000 +0200 @@ -262,6 +262,8 @@ #ifdef CONFIG_HARDLOCKUP_DETECTOR @@ -23456,9 +47726,9 @@ diff -Nur linux-4.1.6.orig/kernel/watchdog.c linux-4.1.6/kernel/watchdog.c /* Enable the perf event */ watchdog_nmi_enable(cpu); -diff -Nur linux-4.1.6.orig/kernel/workqueue.c linux-4.1.6/kernel/workqueue.c ---- linux-4.1.6.orig/kernel/workqueue.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/workqueue.c 2015-09-08 23:49:08.425831903 +0200 +diff -Nur linux-4.1.10.orig/kernel/workqueue.c linux-4.1.10/kernel/workqueue.c +--- linux-4.1.10.orig/kernel/workqueue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/workqueue.c 2015-10-07 18:00:08.000000000 +0200 @@ -48,6 +48,8 @@ #include <linux/nodemask.h> #include <linux/moduleparam.h> @@ -24090,9 +48360,9 @@ diff -Nur linux-4.1.6.orig/kernel/workqueue.c linux-4.1.6/kernel/workqueue.c return written; } -diff -Nur linux-4.1.6.orig/kernel/workqueue_internal.h linux-4.1.6/kernel/workqueue_internal.h ---- linux-4.1.6.orig/kernel/workqueue_internal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/workqueue_internal.h 2015-09-08 23:49:08.425831903 +0200 +diff -Nur linux-4.1.10.orig/kernel/workqueue_internal.h linux-4.1.10/kernel/workqueue_internal.h +--- linux-4.1.10.orig/kernel/workqueue_internal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/workqueue_internal.h 2015-10-07 18:00:08.000000000 +0200 @@ -43,6 +43,7 @@ unsigned long last_active; /* L: last active timestamp */ unsigned int flags; /* X: flags */ @@ -24111,9 +48381,9 @@ diff -Nur linux-4.1.6.orig/kernel/workqueue_internal.h linux-4.1.6/kernel/workqu +void wq_worker_sleeping(struct task_struct *task); #endif /* _KERNEL_WORKQUEUE_INTERNAL_H */ -diff -Nur linux-4.1.6.orig/lib/debugobjects.c linux-4.1.6/lib/debugobjects.c ---- linux-4.1.6.orig/lib/debugobjects.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/debugobjects.c 2015-09-08 23:49:08.425831903 +0200 +diff -Nur linux-4.1.10.orig/lib/debugobjects.c linux-4.1.10/lib/debugobjects.c +--- linux-4.1.10.orig/lib/debugobjects.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/debugobjects.c 2015-10-07 18:00:08.000000000 +0200 @@ -309,7 +309,10 @@ struct debug_obj *obj; unsigned long flags; @@ -24126,9 +48396,9 @@ diff -Nur linux-4.1.6.orig/lib/debugobjects.c linux-4.1.6/lib/debugobjects.c db = get_bucket((unsigned long) addr); -diff -Nur linux-4.1.6.orig/lib/dump_stack.c linux-4.1.6/lib/dump_stack.c ---- linux-4.1.6.orig/lib/dump_stack.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/dump_stack.c 2015-09-08 23:49:08.429831459 +0200 +diff -Nur linux-4.1.10.orig/lib/dump_stack.c linux-4.1.10/lib/dump_stack.c +--- linux-4.1.10.orig/lib/dump_stack.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/dump_stack.c 2015-10-07 18:00:08.000000000 +0200 @@ -33,7 +33,7 @@ * Permit this cpu to perform nested stack dumps while serialising * against other CPUs @@ -24147,9 +48417,9 @@ diff -Nur linux-4.1.6.orig/lib/dump_stack.c linux-4.1.6/lib/dump_stack.c } #else asmlinkage __visible void dump_stack(void) -diff -Nur linux-4.1.6.orig/lib/idr.c linux-4.1.6/lib/idr.c ---- linux-4.1.6.orig/lib/idr.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/idr.c 2015-09-08 23:49:08.429831459 +0200 +diff -Nur linux-4.1.10.orig/lib/idr.c linux-4.1.10/lib/idr.c +--- linux-4.1.10.orig/lib/idr.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/idr.c 2015-10-07 18:00:08.000000000 +0200 @@ -30,6 +30,7 @@ #include <linux/idr.h> #include <linux/spinlock.h> @@ -24215,9 +48485,9 @@ diff -Nur linux-4.1.6.orig/lib/idr.c linux-4.1.6/lib/idr.c if (!new) break; -diff -Nur linux-4.1.6.orig/lib/Kconfig linux-4.1.6/lib/Kconfig ---- linux-4.1.6.orig/lib/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/Kconfig 2015-09-08 23:49:08.425831903 +0200 +diff -Nur linux-4.1.10.orig/lib/Kconfig linux-4.1.10/lib/Kconfig +--- linux-4.1.10.orig/lib/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/Kconfig 2015-10-07 18:00:08.000000000 +0200 @@ -391,6 +391,7 @@ config CPUMASK_OFFSTACK @@ -24226,9 +48496,9 @@ diff -Nur linux-4.1.6.orig/lib/Kconfig linux-4.1.6/lib/Kconfig help Use dynamic allocation for cpumask_var_t, instead of putting them on the stack. This is a bit more expensive, but avoids -diff -Nur linux-4.1.6.orig/lib/locking-selftest.c linux-4.1.6/lib/locking-selftest.c ---- linux-4.1.6.orig/lib/locking-selftest.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/locking-selftest.c 2015-09-08 23:49:08.429831459 +0200 +diff -Nur linux-4.1.10.orig/lib/locking-selftest.c linux-4.1.10/lib/locking-selftest.c +--- linux-4.1.10.orig/lib/locking-selftest.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/locking-selftest.c 2015-10-07 18:00:08.000000000 +0200 @@ -590,6 +590,8 @@ #include "locking-selftest-spin-hardirq.h" GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin) @@ -24377,9 +48647,9 @@ diff -Nur linux-4.1.6.orig/lib/locking-selftest.c linux-4.1.6/lib/locking-selfte ww_tests(); -diff -Nur linux-4.1.6.orig/lib/percpu_ida.c linux-4.1.6/lib/percpu_ida.c ---- linux-4.1.6.orig/lib/percpu_ida.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/percpu_ida.c 2015-09-08 23:49:08.433831016 +0200 +diff -Nur linux-4.1.10.orig/lib/percpu_ida.c linux-4.1.10/lib/percpu_ida.c +--- linux-4.1.10.orig/lib/percpu_ida.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/percpu_ida.c 2015-10-07 18:00:08.000000000 +0200 @@ -26,6 +26,9 @@ #include <linux/string.h> #include <linux/spinlock.h> @@ -24468,9 +48738,9 @@ diff -Nur linux-4.1.6.orig/lib/percpu_ida.c linux-4.1.6/lib/percpu_ida.c return err; } EXPORT_SYMBOL_GPL(percpu_ida_for_each_free); -diff -Nur linux-4.1.6.orig/lib/radix-tree.c linux-4.1.6/lib/radix-tree.c ---- linux-4.1.6.orig/lib/radix-tree.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/radix-tree.c 2015-09-08 23:49:08.433831016 +0200 +diff -Nur linux-4.1.10.orig/lib/radix-tree.c linux-4.1.10/lib/radix-tree.c +--- linux-4.1.10.orig/lib/radix-tree.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/radix-tree.c 2015-10-07 18:00:08.000000000 +0200 @@ -195,12 +195,13 @@ * succeed in getting a node here (and never reach * kmem_cache_alloc) @@ -24502,9 +48772,9 @@ diff -Nur linux-4.1.6.orig/lib/radix-tree.c linux-4.1.6/lib/radix-tree.c /* * Return the maximum key which can be store into a -diff -Nur linux-4.1.6.orig/lib/scatterlist.c linux-4.1.6/lib/scatterlist.c ---- linux-4.1.6.orig/lib/scatterlist.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/scatterlist.c 2015-09-08 23:49:08.433831016 +0200 +diff -Nur linux-4.1.10.orig/lib/scatterlist.c linux-4.1.10/lib/scatterlist.c +--- linux-4.1.10.orig/lib/scatterlist.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/scatterlist.c 2015-10-07 18:00:08.000000000 +0200 @@ -592,7 +592,7 @@ flush_kernel_dcache_page(miter->page); @@ -24532,9 +48802,9 @@ diff -Nur linux-4.1.6.orig/lib/scatterlist.c linux-4.1.6/lib/scatterlist.c return offset; } -diff -Nur linux-4.1.6.orig/lib/smp_processor_id.c linux-4.1.6/lib/smp_processor_id.c ---- linux-4.1.6.orig/lib/smp_processor_id.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/smp_processor_id.c 2015-09-08 23:49:08.433831016 +0200 +diff -Nur linux-4.1.10.orig/lib/smp_processor_id.c linux-4.1.10/lib/smp_processor_id.c +--- linux-4.1.10.orig/lib/smp_processor_id.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/smp_processor_id.c 2015-10-07 18:00:08.000000000 +0200 @@ -39,8 +39,9 @@ if (!printk_ratelimit()) goto out_enable; @@ -24547,9 +48817,9 @@ diff -Nur linux-4.1.6.orig/lib/smp_processor_id.c linux-4.1.6/lib/smp_processor_ print_symbol("caller is %s\n", (long)__builtin_return_address(0)); dump_stack(); -diff -Nur linux-4.1.6.orig/lib/strnlen_user.c linux-4.1.6/lib/strnlen_user.c ---- linux-4.1.6.orig/lib/strnlen_user.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/strnlen_user.c 2015-09-08 23:49:08.433831016 +0200 +diff -Nur linux-4.1.10.orig/lib/strnlen_user.c linux-4.1.10/lib/strnlen_user.c +--- linux-4.1.10.orig/lib/strnlen_user.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/strnlen_user.c 2015-10-07 18:00:08.000000000 +0200 @@ -85,7 +85,8 @@ * @str: The string to measure. * @count: Maximum count (including NUL character) @@ -24570,9 +48840,14 @@ diff -Nur linux-4.1.6.orig/lib/strnlen_user.c linux-4.1.6/lib/strnlen_user.c * * Get the size of a NUL-terminated string in user space. * -diff -Nur linux-4.1.6.orig/mm/compaction.c linux-4.1.6/mm/compaction.c ---- linux-4.1.6.orig/mm/compaction.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/compaction.c 2015-09-08 23:49:08.437830574 +0200 +diff -Nur linux-4.1.10.orig/localversion-rt linux-4.1.10/localversion-rt +--- linux-4.1.10.orig/localversion-rt 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/localversion-rt 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1 @@ ++-rt8 +diff -Nur linux-4.1.10.orig/mm/compaction.c linux-4.1.10/mm/compaction.c +--- linux-4.1.10.orig/mm/compaction.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/compaction.c 2015-10-07 18:00:08.000000000 +0200 @@ -1406,10 +1406,12 @@ cc->migrate_pfn & ~((1UL << cc->order) - 1); @@ -24588,9 +48863,9 @@ diff -Nur linux-4.1.6.orig/mm/compaction.c linux-4.1.6/mm/compaction.c /* No more flushing until we migrate again */ last_migrated_pfn = 0; } -diff -Nur linux-4.1.6.orig/mm/filemap.c linux-4.1.6/mm/filemap.c ---- linux-4.1.6.orig/mm/filemap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/filemap.c 2015-09-08 23:49:08.437830574 +0200 +diff -Nur linux-4.1.10.orig/mm/filemap.c linux-4.1.10/mm/filemap.c +--- linux-4.1.10.orig/mm/filemap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/filemap.c 2015-10-07 18:00:08.000000000 +0200 @@ -167,7 +167,9 @@ if (!workingset_node_pages(node) && list_empty(&node->private_list)) { @@ -24617,9 +48892,9 @@ diff -Nur linux-4.1.6.orig/mm/filemap.c linux-4.1.6/mm/filemap.c } return 0; } -diff -Nur linux-4.1.6.orig/mm/highmem.c linux-4.1.6/mm/highmem.c ---- linux-4.1.6.orig/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/highmem.c 2015-09-08 23:49:08.437830574 +0200 +diff -Nur linux-4.1.10.orig/mm/highmem.c linux-4.1.10/mm/highmem.c +--- linux-4.1.10.orig/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/highmem.c 2015-10-07 18:00:08.000000000 +0200 @@ -29,10 +29,11 @@ #include <linux/kgdb.h> #include <asm/tlbflush.h> @@ -24644,9 +48919,9 @@ diff -Nur linux-4.1.6.orig/mm/highmem.c linux-4.1.6/mm/highmem.c unsigned int nr_free_highpages (void) { -diff -Nur linux-4.1.6.orig/mm/Kconfig linux-4.1.6/mm/Kconfig ---- linux-4.1.6.orig/mm/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/Kconfig 2015-09-08 23:49:08.437830574 +0200 +diff -Nur linux-4.1.10.orig/mm/Kconfig linux-4.1.10/mm/Kconfig +--- linux-4.1.10.orig/mm/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/Kconfig 2015-10-07 18:00:08.000000000 +0200 @@ -409,7 +409,7 @@ config TRANSPARENT_HUGEPAGE @@ -24656,9 +48931,9 @@ diff -Nur linux-4.1.6.orig/mm/Kconfig linux-4.1.6/mm/Kconfig select COMPACTION help Transparent Hugepages allows the kernel to use huge pages and -diff -Nur linux-4.1.6.orig/mm/memcontrol.c linux-4.1.6/mm/memcontrol.c ---- linux-4.1.6.orig/mm/memcontrol.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/memcontrol.c 2015-09-08 23:49:08.437830574 +0200 +diff -Nur linux-4.1.10.orig/mm/memcontrol.c linux-4.1.10/mm/memcontrol.c +--- linux-4.1.10.orig/mm/memcontrol.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/memcontrol.c 2015-10-07 18:00:08.000000000 +0200 @@ -66,6 +66,8 @@ #include <net/sock.h> #include <net/ip.h> @@ -24779,9 +49054,9 @@ diff -Nur linux-4.1.6.orig/mm/memcontrol.c linux-4.1.6/mm/memcontrol.c } /** -diff -Nur linux-4.1.6.orig/mm/memory.c linux-4.1.6/mm/memory.c ---- linux-4.1.6.orig/mm/memory.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/memory.c 2015-09-08 23:49:08.437830574 +0200 +diff -Nur linux-4.1.10.orig/mm/memory.c linux-4.1.10/mm/memory.c +--- linux-4.1.10.orig/mm/memory.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/memory.c 2015-10-07 18:00:08.000000000 +0200 @@ -3743,7 +3743,7 @@ } @@ -24818,9 +49093,9 @@ diff -Nur linux-4.1.6.orig/mm/memory.c linux-4.1.6/mm/memory.c #endif #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) -diff -Nur linux-4.1.6.orig/mm/mmu_context.c linux-4.1.6/mm/mmu_context.c ---- linux-4.1.6.orig/mm/mmu_context.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/mmu_context.c 2015-09-08 23:49:08.437830574 +0200 +diff -Nur linux-4.1.10.orig/mm/mmu_context.c linux-4.1.10/mm/mmu_context.c +--- linux-4.1.10.orig/mm/mmu_context.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/mmu_context.c 2015-10-07 18:00:08.000000000 +0200 @@ -23,6 +23,7 @@ struct task_struct *tsk = current; @@ -24837,9 +49112,9 @@ diff -Nur linux-4.1.6.orig/mm/mmu_context.c linux-4.1.6/mm/mmu_context.c task_unlock(tsk); #ifdef finish_arch_post_lock_switch finish_arch_post_lock_switch(); -diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c ---- linux-4.1.6.orig/mm/page_alloc.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/page_alloc.c 2015-09-08 23:49:08.437830574 +0200 +diff -Nur linux-4.1.10.orig/mm/page_alloc.c linux-4.1.10/mm/page_alloc.c +--- linux-4.1.10.orig/mm/page_alloc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/page_alloc.c 2015-10-07 18:00:08.000000000 +0200 @@ -60,6 +60,7 @@ #include <linux/page_ext.h> #include <linux/hugetlb.h> @@ -24999,7 +49274,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c } void __init __free_pages_bootmem(struct page *page, unsigned int order) -@@ -1368,16 +1406,18 @@ +@@ -1371,16 +1409,18 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) { unsigned long flags; @@ -25021,7 +49296,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c } #endif -@@ -1393,16 +1433,21 @@ +@@ -1396,16 +1436,21 @@ unsigned long flags; struct per_cpu_pageset *pset; struct per_cpu_pages *pcp; @@ -25047,7 +49322,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c } /* -@@ -1488,8 +1533,17 @@ +@@ -1491,8 +1536,17 @@ else cpumask_clear_cpu(cpu, &cpus_with_pcps); } @@ -25065,7 +49340,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c } #ifdef CONFIG_HIBERNATION -@@ -1545,7 +1599,7 @@ +@@ -1548,7 +1602,7 @@ migratetype = get_pfnblock_migratetype(page, pfn); set_freepage_migratetype(page, migratetype); @@ -25074,7 +49349,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c __count_vm_event(PGFREE); /* -@@ -1571,12 +1625,17 @@ +@@ -1574,12 +1628,17 @@ pcp->count++; if (pcp->count >= pcp->high) { unsigned long batch = READ_ONCE(pcp->batch); @@ -25094,7 +49369,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c } /* -@@ -1707,7 +1766,7 @@ +@@ -1710,7 +1769,7 @@ struct per_cpu_pages *pcp; struct list_head *list; @@ -25103,7 +49378,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c pcp = &this_cpu_ptr(zone->pageset)->pcp; list = &pcp->lists[migratetype]; if (list_empty(list)) { -@@ -1739,13 +1798,15 @@ +@@ -1742,13 +1801,15 @@ */ WARN_ON_ONCE(order > 1); } @@ -25122,7 +49397,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c } __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); -@@ -1755,13 +1816,13 @@ +@@ -1758,13 +1819,13 @@ __count_zone_vm_events(PGALLOC, zone, 1 << order); zone_statistics(preferred_zone, zone, gfp_flags); @@ -25138,7 +49413,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c return NULL; } -@@ -5650,6 +5711,7 @@ +@@ -5653,6 +5714,7 @@ void __init page_alloc_init(void) { hotcpu_notifier(page_alloc_cpu_notify, 0); @@ -25146,7 +49421,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c } /* -@@ -6544,7 +6606,7 @@ +@@ -6547,7 +6609,7 @@ struct per_cpu_pageset *pset; /* avoid races with drain_pages() */ @@ -25155,7 +49430,7 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c if (zone->pageset != &boot_pageset) { for_each_online_cpu(cpu) { pset = per_cpu_ptr(zone->pageset, cpu); -@@ -6553,7 +6615,7 @@ +@@ -6556,7 +6618,7 @@ free_percpu(zone->pageset); zone->pageset = &boot_pageset; } @@ -25164,9 +49439,6649 @@ diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c } #ifdef CONFIG_MEMORY_HOTREMOVE -diff -Nur linux-4.1.6.orig/mm/slab.h linux-4.1.6/mm/slab.h ---- linux-4.1.6.orig/mm/slab.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/slab.h 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/mm/page_alloc.c.orig linux-4.1.10/mm/page_alloc.c.orig +--- linux-4.1.10.orig/mm/page_alloc.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/mm/page_alloc.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,6636 @@ ++/* ++ * linux/mm/page_alloc.c ++ * ++ * Manages the free list, the system allocates free pages here. ++ * Note that kmalloc() lives in slab.c ++ * ++ * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds ++ * Swap reorganised 29.12.95, Stephen Tweedie ++ * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 ++ * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 ++ * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 ++ * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 ++ * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 ++ * (lots of bits borrowed from Ingo Molnar & Andrew Morton) ++ */ ++ ++#include <linux/stddef.h> ++#include <linux/mm.h> ++#include <linux/swap.h> ++#include <linux/interrupt.h> ++#include <linux/pagemap.h> ++#include <linux/jiffies.h> ++#include <linux/bootmem.h> ++#include <linux/memblock.h> ++#include <linux/compiler.h> ++#include <linux/kernel.h> ++#include <linux/kmemcheck.h> ++#include <linux/kasan.h> ++#include <linux/module.h> ++#include <linux/suspend.h> ++#include <linux/pagevec.h> ++#include <linux/blkdev.h> ++#include <linux/slab.h> ++#include <linux/ratelimit.h> ++#include <linux/oom.h> ++#include <linux/notifier.h> ++#include <linux/topology.h> ++#include <linux/sysctl.h> ++#include <linux/cpu.h> ++#include <linux/cpuset.h> ++#include <linux/memory_hotplug.h> ++#include <linux/nodemask.h> ++#include <linux/vmalloc.h> ++#include <linux/vmstat.h> ++#include <linux/mempolicy.h> ++#include <linux/stop_machine.h> ++#include <linux/sort.h> ++#include <linux/pfn.h> ++#include <linux/backing-dev.h> ++#include <linux/fault-inject.h> ++#include <linux/page-isolation.h> ++#include <linux/page_ext.h> ++#include <linux/debugobjects.h> ++#include <linux/kmemleak.h> ++#include <linux/compaction.h> ++#include <trace/events/kmem.h> ++#include <linux/prefetch.h> ++#include <linux/mm_inline.h> ++#include <linux/migrate.h> ++#include <linux/page_ext.h> ++#include <linux/hugetlb.h> ++#include <linux/sched/rt.h> ++#include <linux/page_owner.h> ++ ++#include <asm/sections.h> ++#include <asm/tlbflush.h> ++#include <asm/div64.h> ++#include "internal.h" ++ ++/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ ++static DEFINE_MUTEX(pcp_batch_high_lock); ++#define MIN_PERCPU_PAGELIST_FRACTION (8) ++ ++#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID ++DEFINE_PER_CPU(int, numa_node); ++EXPORT_PER_CPU_SYMBOL(numa_node); ++#endif ++ ++#ifdef CONFIG_HAVE_MEMORYLESS_NODES ++/* ++ * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. ++ * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. ++ * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() ++ * defined in <linux/topology.h>. ++ */ ++DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ ++EXPORT_PER_CPU_SYMBOL(_numa_mem_); ++int _node_numa_mem_[MAX_NUMNODES]; ++#endif ++ ++/* ++ * Array of node states. ++ */ ++nodemask_t node_states[NR_NODE_STATES] __read_mostly = { ++ [N_POSSIBLE] = NODE_MASK_ALL, ++ [N_ONLINE] = { { [0] = 1UL } }, ++#ifndef CONFIG_NUMA ++ [N_NORMAL_MEMORY] = { { [0] = 1UL } }, ++#ifdef CONFIG_HIGHMEM ++ [N_HIGH_MEMORY] = { { [0] = 1UL } }, ++#endif ++#ifdef CONFIG_MOVABLE_NODE ++ [N_MEMORY] = { { [0] = 1UL } }, ++#endif ++ [N_CPU] = { { [0] = 1UL } }, ++#endif /* NUMA */ ++}; ++EXPORT_SYMBOL(node_states); ++ ++/* Protect totalram_pages and zone->managed_pages */ ++static DEFINE_SPINLOCK(managed_page_count_lock); ++ ++unsigned long totalram_pages __read_mostly; ++unsigned long totalreserve_pages __read_mostly; ++unsigned long totalcma_pages __read_mostly; ++/* ++ * When calculating the number of globally allowed dirty pages, there ++ * is a certain number of per-zone reserves that should not be ++ * considered dirtyable memory. This is the sum of those reserves ++ * over all existing zones that contribute dirtyable memory. ++ */ ++unsigned long dirty_balance_reserve __read_mostly; ++ ++int percpu_pagelist_fraction; ++gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; ++ ++#ifdef CONFIG_PM_SLEEP ++/* ++ * The following functions are used by the suspend/hibernate code to temporarily ++ * change gfp_allowed_mask in order to avoid using I/O during memory allocations ++ * while devices are suspended. To avoid races with the suspend/hibernate code, ++ * they should always be called with pm_mutex held (gfp_allowed_mask also should ++ * only be modified with pm_mutex held, unless the suspend/hibernate code is ++ * guaranteed not to run in parallel with that modification). ++ */ ++ ++static gfp_t saved_gfp_mask; ++ ++void pm_restore_gfp_mask(void) ++{ ++ WARN_ON(!mutex_is_locked(&pm_mutex)); ++ if (saved_gfp_mask) { ++ gfp_allowed_mask = saved_gfp_mask; ++ saved_gfp_mask = 0; ++ } ++} ++ ++void pm_restrict_gfp_mask(void) ++{ ++ WARN_ON(!mutex_is_locked(&pm_mutex)); ++ WARN_ON(saved_gfp_mask); ++ saved_gfp_mask = gfp_allowed_mask; ++ gfp_allowed_mask &= ~GFP_IOFS; ++} ++ ++bool pm_suspended_storage(void) ++{ ++ if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS) ++ return false; ++ return true; ++} ++#endif /* CONFIG_PM_SLEEP */ ++ ++#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE ++int pageblock_order __read_mostly; ++#endif ++ ++static void __free_pages_ok(struct page *page, unsigned int order); ++ ++/* ++ * results with 256, 32 in the lowmem_reserve sysctl: ++ * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) ++ * 1G machine -> (16M dma, 784M normal, 224M high) ++ * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA ++ * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL ++ * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA ++ * ++ * TBD: should special case ZONE_DMA32 machines here - in those we normally ++ * don't need any ZONE_NORMAL reservation ++ */ ++int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { ++#ifdef CONFIG_ZONE_DMA ++ 256, ++#endif ++#ifdef CONFIG_ZONE_DMA32 ++ 256, ++#endif ++#ifdef CONFIG_HIGHMEM ++ 32, ++#endif ++ 32, ++}; ++ ++EXPORT_SYMBOL(totalram_pages); ++ ++static char * const zone_names[MAX_NR_ZONES] = { ++#ifdef CONFIG_ZONE_DMA ++ "DMA", ++#endif ++#ifdef CONFIG_ZONE_DMA32 ++ "DMA32", ++#endif ++ "Normal", ++#ifdef CONFIG_HIGHMEM ++ "HighMem", ++#endif ++ "Movable", ++}; ++ ++int min_free_kbytes = 1024; ++int user_min_free_kbytes = -1; ++ ++static unsigned long __meminitdata nr_kernel_pages; ++static unsigned long __meminitdata nr_all_pages; ++static unsigned long __meminitdata dma_reserve; ++ ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; ++static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES]; ++static unsigned long __initdata required_kernelcore; ++static unsigned long __initdata required_movablecore; ++static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES]; ++ ++/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ ++int movable_zone; ++EXPORT_SYMBOL(movable_zone); ++#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++ ++#if MAX_NUMNODES > 1 ++int nr_node_ids __read_mostly = MAX_NUMNODES; ++int nr_online_nodes __read_mostly = 1; ++EXPORT_SYMBOL(nr_node_ids); ++EXPORT_SYMBOL(nr_online_nodes); ++#endif ++ ++int page_group_by_mobility_disabled __read_mostly; ++ ++void set_pageblock_migratetype(struct page *page, int migratetype) ++{ ++ if (unlikely(page_group_by_mobility_disabled && ++ migratetype < MIGRATE_PCPTYPES)) ++ migratetype = MIGRATE_UNMOVABLE; ++ ++ set_pageblock_flags_group(page, (unsigned long)migratetype, ++ PB_migrate, PB_migrate_end); ++} ++ ++#ifdef CONFIG_DEBUG_VM ++static int page_outside_zone_boundaries(struct zone *zone, struct page *page) ++{ ++ int ret = 0; ++ unsigned seq; ++ unsigned long pfn = page_to_pfn(page); ++ unsigned long sp, start_pfn; ++ ++ do { ++ seq = zone_span_seqbegin(zone); ++ start_pfn = zone->zone_start_pfn; ++ sp = zone->spanned_pages; ++ if (!zone_spans_pfn(zone, pfn)) ++ ret = 1; ++ } while (zone_span_seqretry(zone, seq)); ++ ++ if (ret) ++ pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", ++ pfn, zone_to_nid(zone), zone->name, ++ start_pfn, start_pfn + sp); ++ ++ return ret; ++} ++ ++static int page_is_consistent(struct zone *zone, struct page *page) ++{ ++ if (!pfn_valid_within(page_to_pfn(page))) ++ return 0; ++ if (zone != page_zone(page)) ++ return 0; ++ ++ return 1; ++} ++/* ++ * Temporary debugging check for pages not lying within a given zone. ++ */ ++static int bad_range(struct zone *zone, struct page *page) ++{ ++ if (page_outside_zone_boundaries(zone, page)) ++ return 1; ++ if (!page_is_consistent(zone, page)) ++ return 1; ++ ++ return 0; ++} ++#else ++static inline int bad_range(struct zone *zone, struct page *page) ++{ ++ return 0; ++} ++#endif ++ ++static void bad_page(struct page *page, const char *reason, ++ unsigned long bad_flags) ++{ ++ static unsigned long resume; ++ static unsigned long nr_shown; ++ static unsigned long nr_unshown; ++ ++ /* Don't complain about poisoned pages */ ++ if (PageHWPoison(page)) { ++ page_mapcount_reset(page); /* remove PageBuddy */ ++ return; ++ } ++ ++ /* ++ * Allow a burst of 60 reports, then keep quiet for that minute; ++ * or allow a steady drip of one report per second. ++ */ ++ if (nr_shown == 60) { ++ if (time_before(jiffies, resume)) { ++ nr_unshown++; ++ goto out; ++ } ++ if (nr_unshown) { ++ printk(KERN_ALERT ++ "BUG: Bad page state: %lu messages suppressed\n", ++ nr_unshown); ++ nr_unshown = 0; ++ } ++ nr_shown = 0; ++ } ++ if (nr_shown++ == 0) ++ resume = jiffies + 60 * HZ; ++ ++ printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n", ++ current->comm, page_to_pfn(page)); ++ dump_page_badflags(page, reason, bad_flags); ++ ++ print_modules(); ++ dump_stack(); ++out: ++ /* Leave bad fields for debug, except PageBuddy could make trouble */ ++ page_mapcount_reset(page); /* remove PageBuddy */ ++ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); ++} ++ ++/* ++ * Higher-order pages are called "compound pages". They are structured thusly: ++ * ++ * The first PAGE_SIZE page is called the "head page". ++ * ++ * The remaining PAGE_SIZE pages are called "tail pages". ++ * ++ * All pages have PG_compound set. All tail pages have their ->first_page ++ * pointing at the head page. ++ * ++ * The first tail page's ->lru.next holds the address of the compound page's ++ * put_page() function. Its ->lru.prev holds the order of allocation. ++ * This usage means that zero-order pages may not be compound. ++ */ ++ ++static void free_compound_page(struct page *page) ++{ ++ __free_pages_ok(page, compound_order(page)); ++} ++ ++void prep_compound_page(struct page *page, unsigned long order) ++{ ++ int i; ++ int nr_pages = 1 << order; ++ ++ set_compound_page_dtor(page, free_compound_page); ++ set_compound_order(page, order); ++ __SetPageHead(page); ++ for (i = 1; i < nr_pages; i++) { ++ struct page *p = page + i; ++ set_page_count(p, 0); ++ p->first_page = page; ++ /* Make sure p->first_page is always valid for PageTail() */ ++ smp_wmb(); ++ __SetPageTail(p); ++ } ++} ++ ++static inline void prep_zero_page(struct page *page, unsigned int order, ++ gfp_t gfp_flags) ++{ ++ int i; ++ ++ /* ++ * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO ++ * and __GFP_HIGHMEM from hard or soft interrupt context. ++ */ ++ VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt()); ++ for (i = 0; i < (1 << order); i++) ++ clear_highpage(page + i); ++} ++ ++#ifdef CONFIG_DEBUG_PAGEALLOC ++unsigned int _debug_guardpage_minorder; ++bool _debug_pagealloc_enabled __read_mostly; ++bool _debug_guardpage_enabled __read_mostly; ++ ++static int __init early_debug_pagealloc(char *buf) ++{ ++ if (!buf) ++ return -EINVAL; ++ ++ if (strcmp(buf, "on") == 0) ++ _debug_pagealloc_enabled = true; ++ ++ return 0; ++} ++early_param("debug_pagealloc", early_debug_pagealloc); ++ ++static bool need_debug_guardpage(void) ++{ ++ /* If we don't use debug_pagealloc, we don't need guard page */ ++ if (!debug_pagealloc_enabled()) ++ return false; ++ ++ return true; ++} ++ ++static void init_debug_guardpage(void) ++{ ++ if (!debug_pagealloc_enabled()) ++ return; ++ ++ _debug_guardpage_enabled = true; ++} ++ ++struct page_ext_operations debug_guardpage_ops = { ++ .need = need_debug_guardpage, ++ .init = init_debug_guardpage, ++}; ++ ++static int __init debug_guardpage_minorder_setup(char *buf) ++{ ++ unsigned long res; ++ ++ if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { ++ printk(KERN_ERR "Bad debug_guardpage_minorder value\n"); ++ return 0; ++ } ++ _debug_guardpage_minorder = res; ++ printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res); ++ return 0; ++} ++__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup); ++ ++static inline void set_page_guard(struct zone *zone, struct page *page, ++ unsigned int order, int migratetype) ++{ ++ struct page_ext *page_ext; ++ ++ if (!debug_guardpage_enabled()) ++ return; ++ ++ page_ext = lookup_page_ext(page); ++ __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); ++ ++ INIT_LIST_HEAD(&page->lru); ++ set_page_private(page, order); ++ /* Guard pages are not available for any usage */ ++ __mod_zone_freepage_state(zone, -(1 << order), migratetype); ++} ++ ++static inline void clear_page_guard(struct zone *zone, struct page *page, ++ unsigned int order, int migratetype) ++{ ++ struct page_ext *page_ext; ++ ++ if (!debug_guardpage_enabled()) ++ return; ++ ++ page_ext = lookup_page_ext(page); ++ __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); ++ ++ set_page_private(page, 0); ++ if (!is_migrate_isolate(migratetype)) ++ __mod_zone_freepage_state(zone, (1 << order), migratetype); ++} ++#else ++struct page_ext_operations debug_guardpage_ops = { NULL, }; ++static inline void set_page_guard(struct zone *zone, struct page *page, ++ unsigned int order, int migratetype) {} ++static inline void clear_page_guard(struct zone *zone, struct page *page, ++ unsigned int order, int migratetype) {} ++#endif ++ ++static inline void set_page_order(struct page *page, unsigned int order) ++{ ++ set_page_private(page, order); ++ __SetPageBuddy(page); ++} ++ ++static inline void rmv_page_order(struct page *page) ++{ ++ __ClearPageBuddy(page); ++ set_page_private(page, 0); ++} ++ ++/* ++ * This function checks whether a page is free && is the buddy ++ * we can do coalesce a page and its buddy if ++ * (a) the buddy is not in a hole && ++ * (b) the buddy is in the buddy system && ++ * (c) a page and its buddy have the same order && ++ * (d) a page and its buddy are in the same zone. ++ * ++ * For recording whether a page is in the buddy system, we set ->_mapcount ++ * PAGE_BUDDY_MAPCOUNT_VALUE. ++ * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is ++ * serialized by zone->lock. ++ * ++ * For recording page's order, we use page_private(page). ++ */ ++static inline int page_is_buddy(struct page *page, struct page *buddy, ++ unsigned int order) ++{ ++ if (!pfn_valid_within(page_to_pfn(buddy))) ++ return 0; ++ ++ if (page_is_guard(buddy) && page_order(buddy) == order) { ++ if (page_zone_id(page) != page_zone_id(buddy)) ++ return 0; ++ ++ VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); ++ ++ return 1; ++ } ++ ++ if (PageBuddy(buddy) && page_order(buddy) == order) { ++ /* ++ * zone check is done late to avoid uselessly ++ * calculating zone/node ids for pages that could ++ * never merge. ++ */ ++ if (page_zone_id(page) != page_zone_id(buddy)) ++ return 0; ++ ++ VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); ++ ++ return 1; ++ } ++ return 0; ++} ++ ++/* ++ * Freeing function for a buddy system allocator. ++ * ++ * The concept of a buddy system is to maintain direct-mapped table ++ * (containing bit values) for memory blocks of various "orders". ++ * The bottom level table contains the map for the smallest allocatable ++ * units of memory (here, pages), and each level above it describes ++ * pairs of units from the levels below, hence, "buddies". ++ * At a high level, all that happens here is marking the table entry ++ * at the bottom level available, and propagating the changes upward ++ * as necessary, plus some accounting needed to play nicely with other ++ * parts of the VM system. ++ * At each level, we keep a list of pages, which are heads of continuous ++ * free pages of length of (1 << order) and marked with _mapcount ++ * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page) ++ * field. ++ * So when we are allocating or freeing one, we can derive the state of the ++ * other. That is, if we allocate a small block, and both were ++ * free, the remainder of the region must be split into blocks. ++ * If a block is freed, and its buddy is also free, then this ++ * triggers coalescing into a block of larger size. ++ * ++ * -- nyc ++ */ ++ ++static inline void __free_one_page(struct page *page, ++ unsigned long pfn, ++ struct zone *zone, unsigned int order, ++ int migratetype) ++{ ++ unsigned long page_idx; ++ unsigned long combined_idx; ++ unsigned long uninitialized_var(buddy_idx); ++ struct page *buddy; ++ int max_order = MAX_ORDER; ++ ++ VM_BUG_ON(!zone_is_initialized(zone)); ++ VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); ++ ++ VM_BUG_ON(migratetype == -1); ++ if (is_migrate_isolate(migratetype)) { ++ /* ++ * We restrict max order of merging to prevent merge ++ * between freepages on isolate pageblock and normal ++ * pageblock. Without this, pageblock isolation ++ * could cause incorrect freepage accounting. ++ */ ++ max_order = min(MAX_ORDER, pageblock_order + 1); ++ } else { ++ __mod_zone_freepage_state(zone, 1 << order, migratetype); ++ } ++ ++ page_idx = pfn & ((1 << max_order) - 1); ++ ++ VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); ++ VM_BUG_ON_PAGE(bad_range(zone, page), page); ++ ++ while (order < max_order - 1) { ++ buddy_idx = __find_buddy_index(page_idx, order); ++ buddy = page + (buddy_idx - page_idx); ++ if (!page_is_buddy(page, buddy, order)) ++ break; ++ /* ++ * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, ++ * merge with it and move up one order. ++ */ ++ if (page_is_guard(buddy)) { ++ clear_page_guard(zone, buddy, order, migratetype); ++ } else { ++ list_del(&buddy->lru); ++ zone->free_area[order].nr_free--; ++ rmv_page_order(buddy); ++ } ++ combined_idx = buddy_idx & page_idx; ++ page = page + (combined_idx - page_idx); ++ page_idx = combined_idx; ++ order++; ++ } ++ set_page_order(page, order); ++ ++ /* ++ * If this is not the largest possible page, check if the buddy ++ * of the next-highest order is free. If it is, it's possible ++ * that pages are being freed that will coalesce soon. In case, ++ * that is happening, add the free page to the tail of the list ++ * so it's less likely to be used soon and more likely to be merged ++ * as a higher order page ++ */ ++ if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { ++ struct page *higher_page, *higher_buddy; ++ combined_idx = buddy_idx & page_idx; ++ higher_page = page + (combined_idx - page_idx); ++ buddy_idx = __find_buddy_index(combined_idx, order + 1); ++ higher_buddy = higher_page + (buddy_idx - combined_idx); ++ if (page_is_buddy(higher_page, higher_buddy, order + 1)) { ++ list_add_tail(&page->lru, ++ &zone->free_area[order].free_list[migratetype]); ++ goto out; ++ } ++ } ++ ++ list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); ++out: ++ zone->free_area[order].nr_free++; ++} ++ ++static inline int free_pages_check(struct page *page) ++{ ++ const char *bad_reason = NULL; ++ unsigned long bad_flags = 0; ++ ++ if (unlikely(page_mapcount(page))) ++ bad_reason = "nonzero mapcount"; ++ if (unlikely(page->mapping != NULL)) ++ bad_reason = "non-NULL mapping"; ++ if (unlikely(atomic_read(&page->_count) != 0)) ++ bad_reason = "nonzero _count"; ++ if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { ++ bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; ++ bad_flags = PAGE_FLAGS_CHECK_AT_FREE; ++ } ++#ifdef CONFIG_MEMCG ++ if (unlikely(page->mem_cgroup)) ++ bad_reason = "page still charged to cgroup"; ++#endif ++ if (unlikely(bad_reason)) { ++ bad_page(page, bad_reason, bad_flags); ++ return 1; ++ } ++ page_cpupid_reset_last(page); ++ if (page->flags & PAGE_FLAGS_CHECK_AT_PREP) ++ page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; ++ return 0; ++} ++ ++/* ++ * Frees a number of pages from the PCP lists ++ * Assumes all pages on list are in same zone, and of same order. ++ * count is the number of pages to free. ++ * ++ * If the zone was previously in an "all pages pinned" state then look to ++ * see if this freeing clears that state. ++ * ++ * And clear the zone's pages_scanned counter, to hold off the "all pages are ++ * pinned" detection logic. ++ */ ++static void free_pcppages_bulk(struct zone *zone, int count, ++ struct per_cpu_pages *pcp) ++{ ++ int migratetype = 0; ++ int batch_free = 0; ++ int to_free = count; ++ unsigned long nr_scanned; ++ ++ spin_lock(&zone->lock); ++ nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); ++ if (nr_scanned) ++ __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); ++ ++ while (to_free) { ++ struct page *page; ++ struct list_head *list; ++ ++ /* ++ * Remove pages from lists in a round-robin fashion. A ++ * batch_free count is maintained that is incremented when an ++ * empty list is encountered. This is so more pages are freed ++ * off fuller lists instead of spinning excessively around empty ++ * lists ++ */ ++ do { ++ batch_free++; ++ if (++migratetype == MIGRATE_PCPTYPES) ++ migratetype = 0; ++ list = &pcp->lists[migratetype]; ++ } while (list_empty(list)); ++ ++ /* This is the only non-empty list. Free them all. */ ++ if (batch_free == MIGRATE_PCPTYPES) ++ batch_free = to_free; ++ ++ do { ++ int mt; /* migratetype of the to-be-freed page */ ++ ++ page = list_entry(list->prev, struct page, lru); ++ /* must delete as __free_one_page list manipulates */ ++ list_del(&page->lru); ++ mt = get_freepage_migratetype(page); ++ if (unlikely(has_isolate_pageblock(zone))) ++ mt = get_pageblock_migratetype(page); ++ ++ /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ ++ __free_one_page(page, page_to_pfn(page), zone, 0, mt); ++ trace_mm_page_pcpu_drain(page, 0, mt); ++ } while (--to_free && --batch_free && !list_empty(list)); ++ } ++ spin_unlock(&zone->lock); ++} ++ ++static void free_one_page(struct zone *zone, ++ struct page *page, unsigned long pfn, ++ unsigned int order, ++ int migratetype) ++{ ++ unsigned long nr_scanned; ++ spin_lock(&zone->lock); ++ nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); ++ if (nr_scanned) ++ __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); ++ ++ if (unlikely(has_isolate_pageblock(zone) || ++ is_migrate_isolate(migratetype))) { ++ migratetype = get_pfnblock_migratetype(page, pfn); ++ } ++ __free_one_page(page, pfn, zone, order, migratetype); ++ spin_unlock(&zone->lock); ++} ++ ++static int free_tail_pages_check(struct page *head_page, struct page *page) ++{ ++ if (!IS_ENABLED(CONFIG_DEBUG_VM)) ++ return 0; ++ if (unlikely(!PageTail(page))) { ++ bad_page(page, "PageTail not set", 0); ++ return 1; ++ } ++ if (unlikely(page->first_page != head_page)) { ++ bad_page(page, "first_page not consistent", 0); ++ return 1; ++ } ++ return 0; ++} ++ ++static bool free_pages_prepare(struct page *page, unsigned int order) ++{ ++ bool compound = PageCompound(page); ++ int i, bad = 0; ++ ++ VM_BUG_ON_PAGE(PageTail(page), page); ++ VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); ++ ++ trace_mm_page_free(page, order); ++ kmemcheck_free_shadow(page, order); ++ kasan_free_pages(page, order); ++ ++ if (PageAnon(page)) ++ page->mapping = NULL; ++ bad += free_pages_check(page); ++ for (i = 1; i < (1 << order); i++) { ++ if (compound) ++ bad += free_tail_pages_check(page, page + i); ++ bad += free_pages_check(page + i); ++ } ++ if (bad) ++ return false; ++ ++ reset_page_owner(page, order); ++ ++ if (!PageHighMem(page)) { ++ debug_check_no_locks_freed(page_address(page), ++ PAGE_SIZE << order); ++ debug_check_no_obj_freed(page_address(page), ++ PAGE_SIZE << order); ++ } ++ arch_free_page(page, order); ++ kernel_map_pages(page, 1 << order, 0); ++ ++ return true; ++} ++ ++static void __free_pages_ok(struct page *page, unsigned int order) ++{ ++ unsigned long flags; ++ int migratetype; ++ unsigned long pfn = page_to_pfn(page); ++ ++ if (!free_pages_prepare(page, order)) ++ return; ++ ++ migratetype = get_pfnblock_migratetype(page, pfn); ++ local_irq_save(flags); ++ __count_vm_events(PGFREE, 1 << order); ++ set_freepage_migratetype(page, migratetype); ++ free_one_page(page_zone(page), page, pfn, order, migratetype); ++ local_irq_restore(flags); ++} ++ ++void __init __free_pages_bootmem(struct page *page, unsigned int order) ++{ ++ unsigned int nr_pages = 1 << order; ++ struct page *p = page; ++ unsigned int loop; ++ ++ prefetchw(p); ++ for (loop = 0; loop < (nr_pages - 1); loop++, p++) { ++ prefetchw(p + 1); ++ __ClearPageReserved(p); ++ set_page_count(p, 0); ++ } ++ __ClearPageReserved(p); ++ set_page_count(p, 0); ++ ++ page_zone(page)->managed_pages += nr_pages; ++ set_page_refcounted(page); ++ __free_pages(page, order); ++} ++ ++#ifdef CONFIG_CMA ++/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ ++void __init init_cma_reserved_pageblock(struct page *page) ++{ ++ unsigned i = pageblock_nr_pages; ++ struct page *p = page; ++ ++ do { ++ __ClearPageReserved(p); ++ set_page_count(p, 0); ++ } while (++p, --i); ++ ++ set_pageblock_migratetype(page, MIGRATE_CMA); ++ ++ if (pageblock_order >= MAX_ORDER) { ++ i = pageblock_nr_pages; ++ p = page; ++ do { ++ set_page_refcounted(p); ++ __free_pages(p, MAX_ORDER - 1); ++ p += MAX_ORDER_NR_PAGES; ++ } while (i -= MAX_ORDER_NR_PAGES); ++ } else { ++ set_page_refcounted(page); ++ __free_pages(page, pageblock_order); ++ } ++ ++ adjust_managed_page_count(page, pageblock_nr_pages); ++} ++#endif ++ ++/* ++ * The order of subdivision here is critical for the IO subsystem. ++ * Please do not alter this order without good reasons and regression ++ * testing. Specifically, as large blocks of memory are subdivided, ++ * the order in which smaller blocks are delivered depends on the order ++ * they're subdivided in this function. This is the primary factor ++ * influencing the order in which pages are delivered to the IO ++ * subsystem according to empirical testing, and this is also justified ++ * by considering the behavior of a buddy system containing a single ++ * large block of memory acted on by a series of small allocations. ++ * This behavior is a critical factor in sglist merging's success. ++ * ++ * -- nyc ++ */ ++static inline void expand(struct zone *zone, struct page *page, ++ int low, int high, struct free_area *area, ++ int migratetype) ++{ ++ unsigned long size = 1 << high; ++ ++ while (high > low) { ++ area--; ++ high--; ++ size >>= 1; ++ VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); ++ ++ if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && ++ debug_guardpage_enabled() && ++ high < debug_guardpage_minorder()) { ++ /* ++ * Mark as guard pages (or page), that will allow to ++ * merge back to allocator when buddy will be freed. ++ * Corresponding page table entries will not be touched, ++ * pages will stay not present in virtual address space ++ */ ++ set_page_guard(zone, &page[size], high, migratetype); ++ continue; ++ } ++ list_add(&page[size].lru, &area->free_list[migratetype]); ++ area->nr_free++; ++ set_page_order(&page[size], high); ++ } ++} ++ ++/* ++ * This page is about to be returned from the page allocator ++ */ ++static inline int check_new_page(struct page *page) ++{ ++ const char *bad_reason = NULL; ++ unsigned long bad_flags = 0; ++ ++ if (unlikely(page_mapcount(page))) ++ bad_reason = "nonzero mapcount"; ++ if (unlikely(page->mapping != NULL)) ++ bad_reason = "non-NULL mapping"; ++ if (unlikely(atomic_read(&page->_count) != 0)) ++ bad_reason = "nonzero _count"; ++ if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { ++ bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; ++ bad_flags = PAGE_FLAGS_CHECK_AT_PREP; ++ } ++#ifdef CONFIG_MEMCG ++ if (unlikely(page->mem_cgroup)) ++ bad_reason = "page still charged to cgroup"; ++#endif ++ if (unlikely(bad_reason)) { ++ bad_page(page, bad_reason, bad_flags); ++ return 1; ++ } ++ return 0; ++} ++ ++static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, ++ int alloc_flags) ++{ ++ int i; ++ ++ for (i = 0; i < (1 << order); i++) { ++ struct page *p = page + i; ++ if (unlikely(check_new_page(p))) ++ return 1; ++ } ++ ++ set_page_private(page, 0); ++ set_page_refcounted(page); ++ ++ arch_alloc_page(page, order); ++ kernel_map_pages(page, 1 << order, 1); ++ kasan_alloc_pages(page, order); ++ ++ if (gfp_flags & __GFP_ZERO) ++ prep_zero_page(page, order, gfp_flags); ++ ++ if (order && (gfp_flags & __GFP_COMP)) ++ prep_compound_page(page, order); ++ ++ set_page_owner(page, order, gfp_flags); ++ ++ /* ++ * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to ++ * allocate the page. The expectation is that the caller is taking ++ * steps that will free more memory. The caller should avoid the page ++ * being used for !PFMEMALLOC purposes. ++ */ ++ if (alloc_flags & ALLOC_NO_WATERMARKS) ++ set_page_pfmemalloc(page); ++ else ++ clear_page_pfmemalloc(page); ++ ++ return 0; ++} ++ ++/* ++ * Go through the free lists for the given migratetype and remove ++ * the smallest available page from the freelists ++ */ ++static inline ++struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, ++ int migratetype) ++{ ++ unsigned int current_order; ++ struct free_area *area; ++ struct page *page; ++ ++ /* Find a page of the appropriate size in the preferred list */ ++ for (current_order = order; current_order < MAX_ORDER; ++current_order) { ++ area = &(zone->free_area[current_order]); ++ if (list_empty(&area->free_list[migratetype])) ++ continue; ++ ++ page = list_entry(area->free_list[migratetype].next, ++ struct page, lru); ++ list_del(&page->lru); ++ rmv_page_order(page); ++ area->nr_free--; ++ expand(zone, page, order, current_order, area, migratetype); ++ set_freepage_migratetype(page, migratetype); ++ return page; ++ } ++ ++ return NULL; ++} ++ ++ ++/* ++ * This array describes the order lists are fallen back to when ++ * the free lists for the desirable migrate type are depleted ++ */ ++static int fallbacks[MIGRATE_TYPES][4] = { ++ [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE }, ++ [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE }, ++ [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE }, ++#ifdef CONFIG_CMA ++ [MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */ ++#endif ++ [MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */ ++#ifdef CONFIG_MEMORY_ISOLATION ++ [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */ ++#endif ++}; ++ ++#ifdef CONFIG_CMA ++static struct page *__rmqueue_cma_fallback(struct zone *zone, ++ unsigned int order) ++{ ++ return __rmqueue_smallest(zone, order, MIGRATE_CMA); ++} ++#else ++static inline struct page *__rmqueue_cma_fallback(struct zone *zone, ++ unsigned int order) { return NULL; } ++#endif ++ ++/* ++ * Move the free pages in a range to the free lists of the requested type. ++ * Note that start_page and end_pages are not aligned on a pageblock ++ * boundary. If alignment is required, use move_freepages_block() ++ */ ++int move_freepages(struct zone *zone, ++ struct page *start_page, struct page *end_page, ++ int migratetype) ++{ ++ struct page *page; ++ unsigned long order; ++ int pages_moved = 0; ++ ++#ifndef CONFIG_HOLES_IN_ZONE ++ /* ++ * page_zone is not safe to call in this context when ++ * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant ++ * anyway as we check zone boundaries in move_freepages_block(). ++ * Remove at a later date when no bug reports exist related to ++ * grouping pages by mobility ++ */ ++ VM_BUG_ON(page_zone(start_page) != page_zone(end_page)); ++#endif ++ ++ for (page = start_page; page <= end_page;) { ++ /* Make sure we are not inadvertently changing nodes */ ++ VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); ++ ++ if (!pfn_valid_within(page_to_pfn(page))) { ++ page++; ++ continue; ++ } ++ ++ if (!PageBuddy(page)) { ++ page++; ++ continue; ++ } ++ ++ order = page_order(page); ++ list_move(&page->lru, ++ &zone->free_area[order].free_list[migratetype]); ++ set_freepage_migratetype(page, migratetype); ++ page += 1 << order; ++ pages_moved += 1 << order; ++ } ++ ++ return pages_moved; ++} ++ ++int move_freepages_block(struct zone *zone, struct page *page, ++ int migratetype) ++{ ++ unsigned long start_pfn, end_pfn; ++ struct page *start_page, *end_page; ++ ++ start_pfn = page_to_pfn(page); ++ start_pfn = start_pfn & ~(pageblock_nr_pages-1); ++ start_page = pfn_to_page(start_pfn); ++ end_page = start_page + pageblock_nr_pages - 1; ++ end_pfn = start_pfn + pageblock_nr_pages - 1; ++ ++ /* Do not cross zone boundaries */ ++ if (!zone_spans_pfn(zone, start_pfn)) ++ start_page = page; ++ if (!zone_spans_pfn(zone, end_pfn)) ++ return 0; ++ ++ return move_freepages(zone, start_page, end_page, migratetype); ++} ++ ++static void change_pageblock_range(struct page *pageblock_page, ++ int start_order, int migratetype) ++{ ++ int nr_pageblocks = 1 << (start_order - pageblock_order); ++ ++ while (nr_pageblocks--) { ++ set_pageblock_migratetype(pageblock_page, migratetype); ++ pageblock_page += pageblock_nr_pages; ++ } ++} ++ ++/* ++ * When we are falling back to another migratetype during allocation, try to ++ * steal extra free pages from the same pageblocks to satisfy further ++ * allocations, instead of polluting multiple pageblocks. ++ * ++ * If we are stealing a relatively large buddy page, it is likely there will ++ * be more free pages in the pageblock, so try to steal them all. For ++ * reclaimable and unmovable allocations, we steal regardless of page size, ++ * as fragmentation caused by those allocations polluting movable pageblocks ++ * is worse than movable allocations stealing from unmovable and reclaimable ++ * pageblocks. ++ */ ++static bool can_steal_fallback(unsigned int order, int start_mt) ++{ ++ /* ++ * Leaving this order check is intended, although there is ++ * relaxed order check in next check. The reason is that ++ * we can actually steal whole pageblock if this condition met, ++ * but, below check doesn't guarantee it and that is just heuristic ++ * so could be changed anytime. ++ */ ++ if (order >= pageblock_order) ++ return true; ++ ++ if (order >= pageblock_order / 2 || ++ start_mt == MIGRATE_RECLAIMABLE || ++ start_mt == MIGRATE_UNMOVABLE || ++ page_group_by_mobility_disabled) ++ return true; ++ ++ return false; ++} ++ ++/* ++ * This function implements actual steal behaviour. If order is large enough, ++ * we can steal whole pageblock. If not, we first move freepages in this ++ * pageblock and check whether half of pages are moved or not. If half of ++ * pages are moved, we can change migratetype of pageblock and permanently ++ * use it's pages as requested migratetype in the future. ++ */ ++static void steal_suitable_fallback(struct zone *zone, struct page *page, ++ int start_type) ++{ ++ int current_order = page_order(page); ++ int pages; ++ ++ /* Take ownership for orders >= pageblock_order */ ++ if (current_order >= pageblock_order) { ++ change_pageblock_range(page, current_order, start_type); ++ return; ++ } ++ ++ pages = move_freepages_block(zone, page, start_type); ++ ++ /* Claim the whole block if over half of it is free */ ++ if (pages >= (1 << (pageblock_order-1)) || ++ page_group_by_mobility_disabled) ++ set_pageblock_migratetype(page, start_type); ++} ++ ++/* ++ * Check whether there is a suitable fallback freepage with requested order. ++ * If only_stealable is true, this function returns fallback_mt only if ++ * we can steal other freepages all together. This would help to reduce ++ * fragmentation due to mixed migratetype pages in one pageblock. ++ */ ++int find_suitable_fallback(struct free_area *area, unsigned int order, ++ int migratetype, bool only_stealable, bool *can_steal) ++{ ++ int i; ++ int fallback_mt; ++ ++ if (area->nr_free == 0) ++ return -1; ++ ++ *can_steal = false; ++ for (i = 0;; i++) { ++ fallback_mt = fallbacks[migratetype][i]; ++ if (fallback_mt == MIGRATE_RESERVE) ++ break; ++ ++ if (list_empty(&area->free_list[fallback_mt])) ++ continue; ++ ++ if (can_steal_fallback(order, migratetype)) ++ *can_steal = true; ++ ++ if (!only_stealable) ++ return fallback_mt; ++ ++ if (*can_steal) ++ return fallback_mt; ++ } ++ ++ return -1; ++} ++ ++/* Remove an element from the buddy allocator from the fallback list */ ++static inline struct page * ++__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) ++{ ++ struct free_area *area; ++ unsigned int current_order; ++ struct page *page; ++ int fallback_mt; ++ bool can_steal; ++ ++ /* Find the largest possible block of pages in the other list */ ++ for (current_order = MAX_ORDER-1; ++ current_order >= order && current_order <= MAX_ORDER-1; ++ --current_order) { ++ area = &(zone->free_area[current_order]); ++ fallback_mt = find_suitable_fallback(area, current_order, ++ start_migratetype, false, &can_steal); ++ if (fallback_mt == -1) ++ continue; ++ ++ page = list_entry(area->free_list[fallback_mt].next, ++ struct page, lru); ++ if (can_steal) ++ steal_suitable_fallback(zone, page, start_migratetype); ++ ++ /* Remove the page from the freelists */ ++ area->nr_free--; ++ list_del(&page->lru); ++ rmv_page_order(page); ++ ++ expand(zone, page, order, current_order, area, ++ start_migratetype); ++ /* ++ * The freepage_migratetype may differ from pageblock's ++ * migratetype depending on the decisions in ++ * try_to_steal_freepages(). This is OK as long as it ++ * does not differ for MIGRATE_CMA pageblocks. For CMA ++ * we need to make sure unallocated pages flushed from ++ * pcp lists are returned to the correct freelist. ++ */ ++ set_freepage_migratetype(page, start_migratetype); ++ ++ trace_mm_page_alloc_extfrag(page, order, current_order, ++ start_migratetype, fallback_mt); ++ ++ return page; ++ } ++ ++ return NULL; ++} ++ ++/* ++ * Do the hard work of removing an element from the buddy allocator. ++ * Call me with the zone->lock already held. ++ */ ++static struct page *__rmqueue(struct zone *zone, unsigned int order, ++ int migratetype) ++{ ++ struct page *page; ++ ++retry_reserve: ++ page = __rmqueue_smallest(zone, order, migratetype); ++ ++ if (unlikely(!page) && migratetype != MIGRATE_RESERVE) { ++ if (migratetype == MIGRATE_MOVABLE) ++ page = __rmqueue_cma_fallback(zone, order); ++ ++ if (!page) ++ page = __rmqueue_fallback(zone, order, migratetype); ++ ++ /* ++ * Use MIGRATE_RESERVE rather than fail an allocation. goto ++ * is used because __rmqueue_smallest is an inline function ++ * and we want just one call site ++ */ ++ if (!page) { ++ migratetype = MIGRATE_RESERVE; ++ goto retry_reserve; ++ } ++ } ++ ++ trace_mm_page_alloc_zone_locked(page, order, migratetype); ++ return page; ++} ++ ++/* ++ * Obtain a specified number of elements from the buddy allocator, all under ++ * a single hold of the lock, for efficiency. Add them to the supplied list. ++ * Returns the number of new pages which were placed at *list. ++ */ ++static int rmqueue_bulk(struct zone *zone, unsigned int order, ++ unsigned long count, struct list_head *list, ++ int migratetype, bool cold) ++{ ++ int i; ++ ++ spin_lock(&zone->lock); ++ for (i = 0; i < count; ++i) { ++ struct page *page = __rmqueue(zone, order, migratetype); ++ if (unlikely(page == NULL)) ++ break; ++ ++ /* ++ * Split buddy pages returned by expand() are received here ++ * in physical page order. The page is added to the callers and ++ * list and the list head then moves forward. From the callers ++ * perspective, the linked list is ordered by page number in ++ * some conditions. This is useful for IO devices that can ++ * merge IO requests if the physical pages are ordered ++ * properly. ++ */ ++ if (likely(!cold)) ++ list_add(&page->lru, list); ++ else ++ list_add_tail(&page->lru, list); ++ list = &page->lru; ++ if (is_migrate_cma(get_freepage_migratetype(page))) ++ __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, ++ -(1 << order)); ++ } ++ __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); ++ spin_unlock(&zone->lock); ++ return i; ++} ++ ++#ifdef CONFIG_NUMA ++/* ++ * Called from the vmstat counter updater to drain pagesets of this ++ * currently executing processor on remote nodes after they have ++ * expired. ++ * ++ * Note that this function must be called with the thread pinned to ++ * a single processor. ++ */ ++void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) ++{ ++ unsigned long flags; ++ int to_drain, batch; ++ ++ local_irq_save(flags); ++ batch = READ_ONCE(pcp->batch); ++ to_drain = min(pcp->count, batch); ++ if (to_drain > 0) { ++ free_pcppages_bulk(zone, to_drain, pcp); ++ pcp->count -= to_drain; ++ } ++ local_irq_restore(flags); ++} ++#endif ++ ++/* ++ * Drain pcplists of the indicated processor and zone. ++ * ++ * The processor must either be the current processor and the ++ * thread pinned to the current processor or a processor that ++ * is not online. ++ */ ++static void drain_pages_zone(unsigned int cpu, struct zone *zone) ++{ ++ unsigned long flags; ++ struct per_cpu_pageset *pset; ++ struct per_cpu_pages *pcp; ++ ++ local_irq_save(flags); ++ pset = per_cpu_ptr(zone->pageset, cpu); ++ ++ pcp = &pset->pcp; ++ if (pcp->count) { ++ free_pcppages_bulk(zone, pcp->count, pcp); ++ pcp->count = 0; ++ } ++ local_irq_restore(flags); ++} ++ ++/* ++ * Drain pcplists of all zones on the indicated processor. ++ * ++ * The processor must either be the current processor and the ++ * thread pinned to the current processor or a processor that ++ * is not online. ++ */ ++static void drain_pages(unsigned int cpu) ++{ ++ struct zone *zone; ++ ++ for_each_populated_zone(zone) { ++ drain_pages_zone(cpu, zone); ++ } ++} ++ ++/* ++ * Spill all of this CPU's per-cpu pages back into the buddy allocator. ++ * ++ * The CPU has to be pinned. When zone parameter is non-NULL, spill just ++ * the single zone's pages. ++ */ ++void drain_local_pages(struct zone *zone) ++{ ++ int cpu = smp_processor_id(); ++ ++ if (zone) ++ drain_pages_zone(cpu, zone); ++ else ++ drain_pages(cpu); ++} ++ ++/* ++ * Spill all the per-cpu pages from all CPUs back into the buddy allocator. ++ * ++ * When zone parameter is non-NULL, spill just the single zone's pages. ++ * ++ * Note that this code is protected against sending an IPI to an offline ++ * CPU but does not guarantee sending an IPI to newly hotplugged CPUs: ++ * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but ++ * nothing keeps CPUs from showing up after we populated the cpumask and ++ * before the call to on_each_cpu_mask(). ++ */ ++void drain_all_pages(struct zone *zone) ++{ ++ int cpu; ++ ++ /* ++ * Allocate in the BSS so we wont require allocation in ++ * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y ++ */ ++ static cpumask_t cpus_with_pcps; ++ ++ /* ++ * We don't care about racing with CPU hotplug event ++ * as offline notification will cause the notified ++ * cpu to drain that CPU pcps and on_each_cpu_mask ++ * disables preemption as part of its processing ++ */ ++ for_each_online_cpu(cpu) { ++ struct per_cpu_pageset *pcp; ++ struct zone *z; ++ bool has_pcps = false; ++ ++ if (zone) { ++ pcp = per_cpu_ptr(zone->pageset, cpu); ++ if (pcp->pcp.count) ++ has_pcps = true; ++ } else { ++ for_each_populated_zone(z) { ++ pcp = per_cpu_ptr(z->pageset, cpu); ++ if (pcp->pcp.count) { ++ has_pcps = true; ++ break; ++ } ++ } ++ } ++ ++ if (has_pcps) ++ cpumask_set_cpu(cpu, &cpus_with_pcps); ++ else ++ cpumask_clear_cpu(cpu, &cpus_with_pcps); ++ } ++ on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, ++ zone, 1); ++} ++ ++#ifdef CONFIG_HIBERNATION ++ ++void mark_free_pages(struct zone *zone) ++{ ++ unsigned long pfn, max_zone_pfn; ++ unsigned long flags; ++ unsigned int order, t; ++ struct list_head *curr; ++ ++ if (zone_is_empty(zone)) ++ return; ++ ++ spin_lock_irqsave(&zone->lock, flags); ++ ++ max_zone_pfn = zone_end_pfn(zone); ++ for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) ++ if (pfn_valid(pfn)) { ++ struct page *page = pfn_to_page(pfn); ++ ++ if (!swsusp_page_is_forbidden(page)) ++ swsusp_unset_page_free(page); ++ } ++ ++ for_each_migratetype_order(order, t) { ++ list_for_each(curr, &zone->free_area[order].free_list[t]) { ++ unsigned long i; ++ ++ pfn = page_to_pfn(list_entry(curr, struct page, lru)); ++ for (i = 0; i < (1UL << order); i++) ++ swsusp_set_page_free(pfn_to_page(pfn + i)); ++ } ++ } ++ spin_unlock_irqrestore(&zone->lock, flags); ++} ++#endif /* CONFIG_PM */ ++ ++/* ++ * Free a 0-order page ++ * cold == true ? free a cold page : free a hot page ++ */ ++void free_hot_cold_page(struct page *page, bool cold) ++{ ++ struct zone *zone = page_zone(page); ++ struct per_cpu_pages *pcp; ++ unsigned long flags; ++ unsigned long pfn = page_to_pfn(page); ++ int migratetype; ++ ++ if (!free_pages_prepare(page, 0)) ++ return; ++ ++ migratetype = get_pfnblock_migratetype(page, pfn); ++ set_freepage_migratetype(page, migratetype); ++ local_irq_save(flags); ++ __count_vm_event(PGFREE); ++ ++ /* ++ * We only track unmovable, reclaimable and movable on pcp lists. ++ * Free ISOLATE pages back to the allocator because they are being ++ * offlined but treat RESERVE as movable pages so we can get those ++ * areas back if necessary. Otherwise, we may have to free ++ * excessively into the page allocator ++ */ ++ if (migratetype >= MIGRATE_PCPTYPES) { ++ if (unlikely(is_migrate_isolate(migratetype))) { ++ free_one_page(zone, page, pfn, 0, migratetype); ++ goto out; ++ } ++ migratetype = MIGRATE_MOVABLE; ++ } ++ ++ pcp = &this_cpu_ptr(zone->pageset)->pcp; ++ if (!cold) ++ list_add(&page->lru, &pcp->lists[migratetype]); ++ else ++ list_add_tail(&page->lru, &pcp->lists[migratetype]); ++ pcp->count++; ++ if (pcp->count >= pcp->high) { ++ unsigned long batch = READ_ONCE(pcp->batch); ++ free_pcppages_bulk(zone, batch, pcp); ++ pcp->count -= batch; ++ } ++ ++out: ++ local_irq_restore(flags); ++} ++ ++/* ++ * Free a list of 0-order pages ++ */ ++void free_hot_cold_page_list(struct list_head *list, bool cold) ++{ ++ struct page *page, *next; ++ ++ list_for_each_entry_safe(page, next, list, lru) { ++ trace_mm_page_free_batched(page, cold); ++ free_hot_cold_page(page, cold); ++ } ++} ++ ++/* ++ * split_page takes a non-compound higher-order page, and splits it into ++ * n (1<<order) sub-pages: page[0..n] ++ * Each sub-page must be freed individually. ++ * ++ * Note: this is probably too low level an operation for use in drivers. ++ * Please consult with lkml before using this in your driver. ++ */ ++void split_page(struct page *page, unsigned int order) ++{ ++ int i; ++ ++ VM_BUG_ON_PAGE(PageCompound(page), page); ++ VM_BUG_ON_PAGE(!page_count(page), page); ++ ++#ifdef CONFIG_KMEMCHECK ++ /* ++ * Split shadow pages too, because free(page[0]) would ++ * otherwise free the whole shadow. ++ */ ++ if (kmemcheck_page_is_tracked(page)) ++ split_page(virt_to_page(page[0].shadow), order); ++#endif ++ ++ set_page_owner(page, 0, 0); ++ for (i = 1; i < (1 << order); i++) { ++ set_page_refcounted(page + i); ++ set_page_owner(page + i, 0, 0); ++ } ++} ++EXPORT_SYMBOL_GPL(split_page); ++ ++int __isolate_free_page(struct page *page, unsigned int order) ++{ ++ unsigned long watermark; ++ struct zone *zone; ++ int mt; ++ ++ BUG_ON(!PageBuddy(page)); ++ ++ zone = page_zone(page); ++ mt = get_pageblock_migratetype(page); ++ ++ if (!is_migrate_isolate(mt)) { ++ /* Obey watermarks as if the page was being allocated */ ++ watermark = low_wmark_pages(zone) + (1 << order); ++ if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) ++ return 0; ++ ++ __mod_zone_freepage_state(zone, -(1UL << order), mt); ++ } ++ ++ /* Remove page from free list */ ++ list_del(&page->lru); ++ zone->free_area[order].nr_free--; ++ rmv_page_order(page); ++ ++ /* Set the pageblock if the isolated page is at least a pageblock */ ++ if (order >= pageblock_order - 1) { ++ struct page *endpage = page + (1 << order) - 1; ++ for (; page < endpage; page += pageblock_nr_pages) { ++ int mt = get_pageblock_migratetype(page); ++ if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)) ++ set_pageblock_migratetype(page, ++ MIGRATE_MOVABLE); ++ } ++ } ++ ++ set_page_owner(page, order, 0); ++ return 1UL << order; ++} ++ ++/* ++ * Similar to split_page except the page is already free. As this is only ++ * being used for migration, the migratetype of the block also changes. ++ * As this is called with interrupts disabled, the caller is responsible ++ * for calling arch_alloc_page() and kernel_map_page() after interrupts ++ * are enabled. ++ * ++ * Note: this is probably too low level an operation for use in drivers. ++ * Please consult with lkml before using this in your driver. ++ */ ++int split_free_page(struct page *page) ++{ ++ unsigned int order; ++ int nr_pages; ++ ++ order = page_order(page); ++ ++ nr_pages = __isolate_free_page(page, order); ++ if (!nr_pages) ++ return 0; ++ ++ /* Split into individual pages */ ++ set_page_refcounted(page); ++ split_page(page, order); ++ return nr_pages; ++} ++ ++/* ++ * Allocate a page from the given zone. Use pcplists for order-0 allocations. ++ */ ++static inline ++struct page *buffered_rmqueue(struct zone *preferred_zone, ++ struct zone *zone, unsigned int order, ++ gfp_t gfp_flags, int migratetype) ++{ ++ unsigned long flags; ++ struct page *page; ++ bool cold = ((gfp_flags & __GFP_COLD) != 0); ++ ++ if (likely(order == 0)) { ++ struct per_cpu_pages *pcp; ++ struct list_head *list; ++ ++ local_irq_save(flags); ++ pcp = &this_cpu_ptr(zone->pageset)->pcp; ++ list = &pcp->lists[migratetype]; ++ if (list_empty(list)) { ++ pcp->count += rmqueue_bulk(zone, 0, ++ pcp->batch, list, ++ migratetype, cold); ++ if (unlikely(list_empty(list))) ++ goto failed; ++ } ++ ++ if (cold) ++ page = list_entry(list->prev, struct page, lru); ++ else ++ page = list_entry(list->next, struct page, lru); ++ ++ list_del(&page->lru); ++ pcp->count--; ++ } else { ++ if (unlikely(gfp_flags & __GFP_NOFAIL)) { ++ /* ++ * __GFP_NOFAIL is not to be used in new code. ++ * ++ * All __GFP_NOFAIL callers should be fixed so that they ++ * properly detect and handle allocation failures. ++ * ++ * We most definitely don't want callers attempting to ++ * allocate greater than order-1 page units with ++ * __GFP_NOFAIL. ++ */ ++ WARN_ON_ONCE(order > 1); ++ } ++ spin_lock_irqsave(&zone->lock, flags); ++ page = __rmqueue(zone, order, migratetype); ++ spin_unlock(&zone->lock); ++ if (!page) ++ goto failed; ++ __mod_zone_freepage_state(zone, -(1 << order), ++ get_freepage_migratetype(page)); ++ } ++ ++ __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); ++ if (atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 && ++ !test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) ++ set_bit(ZONE_FAIR_DEPLETED, &zone->flags); ++ ++ __count_zone_vm_events(PGALLOC, zone, 1 << order); ++ zone_statistics(preferred_zone, zone, gfp_flags); ++ local_irq_restore(flags); ++ ++ VM_BUG_ON_PAGE(bad_range(zone, page), page); ++ return page; ++ ++failed: ++ local_irq_restore(flags); ++ return NULL; ++} ++ ++#ifdef CONFIG_FAIL_PAGE_ALLOC ++ ++static struct { ++ struct fault_attr attr; ++ ++ u32 ignore_gfp_highmem; ++ u32 ignore_gfp_wait; ++ u32 min_order; ++} fail_page_alloc = { ++ .attr = FAULT_ATTR_INITIALIZER, ++ .ignore_gfp_wait = 1, ++ .ignore_gfp_highmem = 1, ++ .min_order = 1, ++}; ++ ++static int __init setup_fail_page_alloc(char *str) ++{ ++ return setup_fault_attr(&fail_page_alloc.attr, str); ++} ++__setup("fail_page_alloc=", setup_fail_page_alloc); ++ ++static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) ++{ ++ if (order < fail_page_alloc.min_order) ++ return false; ++ if (gfp_mask & __GFP_NOFAIL) ++ return false; ++ if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) ++ return false; ++ if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT)) ++ return false; ++ ++ return should_fail(&fail_page_alloc.attr, 1 << order); ++} ++ ++#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS ++ ++static int __init fail_page_alloc_debugfs(void) ++{ ++ umode_t mode = S_IFREG | S_IRUSR | S_IWUSR; ++ struct dentry *dir; ++ ++ dir = fault_create_debugfs_attr("fail_page_alloc", NULL, ++ &fail_page_alloc.attr); ++ if (IS_ERR(dir)) ++ return PTR_ERR(dir); ++ ++ if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, ++ &fail_page_alloc.ignore_gfp_wait)) ++ goto fail; ++ if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir, ++ &fail_page_alloc.ignore_gfp_highmem)) ++ goto fail; ++ if (!debugfs_create_u32("min-order", mode, dir, ++ &fail_page_alloc.min_order)) ++ goto fail; ++ ++ return 0; ++fail: ++ debugfs_remove_recursive(dir); ++ ++ return -ENOMEM; ++} ++ ++late_initcall(fail_page_alloc_debugfs); ++ ++#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ ++ ++#else /* CONFIG_FAIL_PAGE_ALLOC */ ++ ++static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) ++{ ++ return false; ++} ++ ++#endif /* CONFIG_FAIL_PAGE_ALLOC */ ++ ++/* ++ * Return true if free pages are above 'mark'. This takes into account the order ++ * of the allocation. ++ */ ++static bool __zone_watermark_ok(struct zone *z, unsigned int order, ++ unsigned long mark, int classzone_idx, int alloc_flags, ++ long free_pages) ++{ ++ /* free_pages may go negative - that's OK */ ++ long min = mark; ++ int o; ++ long free_cma = 0; ++ ++ free_pages -= (1 << order) - 1; ++ if (alloc_flags & ALLOC_HIGH) ++ min -= min / 2; ++ if (alloc_flags & ALLOC_HARDER) ++ min -= min / 4; ++#ifdef CONFIG_CMA ++ /* If allocation can't use CMA areas don't use free CMA pages */ ++ if (!(alloc_flags & ALLOC_CMA)) ++ free_cma = zone_page_state(z, NR_FREE_CMA_PAGES); ++#endif ++ ++ if (free_pages - free_cma <= min + z->lowmem_reserve[classzone_idx]) ++ return false; ++ for (o = 0; o < order; o++) { ++ /* At the next order, this order's pages become unavailable */ ++ free_pages -= z->free_area[o].nr_free << o; ++ ++ /* Require fewer higher order pages to be free */ ++ min >>= 1; ++ ++ if (free_pages <= min) ++ return false; ++ } ++ return true; ++} ++ ++bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, ++ int classzone_idx, int alloc_flags) ++{ ++ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, ++ zone_page_state(z, NR_FREE_PAGES)); ++} ++ ++bool zone_watermark_ok_safe(struct zone *z, unsigned int order, ++ unsigned long mark, int classzone_idx, int alloc_flags) ++{ ++ long free_pages = zone_page_state(z, NR_FREE_PAGES); ++ ++ if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) ++ free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); ++ ++ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, ++ free_pages); ++} ++ ++#ifdef CONFIG_NUMA ++/* ++ * zlc_setup - Setup for "zonelist cache". Uses cached zone data to ++ * skip over zones that are not allowed by the cpuset, or that have ++ * been recently (in last second) found to be nearly full. See further ++ * comments in mmzone.h. Reduces cache footprint of zonelist scans ++ * that have to skip over a lot of full or unallowed zones. ++ * ++ * If the zonelist cache is present in the passed zonelist, then ++ * returns a pointer to the allowed node mask (either the current ++ * tasks mems_allowed, or node_states[N_MEMORY].) ++ * ++ * If the zonelist cache is not available for this zonelist, does ++ * nothing and returns NULL. ++ * ++ * If the fullzones BITMAP in the zonelist cache is stale (more than ++ * a second since last zap'd) then we zap it out (clear its bits.) ++ * ++ * We hold off even calling zlc_setup, until after we've checked the ++ * first zone in the zonelist, on the theory that most allocations will ++ * be satisfied from that first zone, so best to examine that zone as ++ * quickly as we can. ++ */ ++static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) ++{ ++ struct zonelist_cache *zlc; /* cached zonelist speedup info */ ++ nodemask_t *allowednodes; /* zonelist_cache approximation */ ++ ++ zlc = zonelist->zlcache_ptr; ++ if (!zlc) ++ return NULL; ++ ++ if (time_after(jiffies, zlc->last_full_zap + HZ)) { ++ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); ++ zlc->last_full_zap = jiffies; ++ } ++ ++ allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ? ++ &cpuset_current_mems_allowed : ++ &node_states[N_MEMORY]; ++ return allowednodes; ++} ++ ++/* ++ * Given 'z' scanning a zonelist, run a couple of quick checks to see ++ * if it is worth looking at further for free memory: ++ * 1) Check that the zone isn't thought to be full (doesn't have its ++ * bit set in the zonelist_cache fullzones BITMAP). ++ * 2) Check that the zones node (obtained from the zonelist_cache ++ * z_to_n[] mapping) is allowed in the passed in allowednodes mask. ++ * Return true (non-zero) if zone is worth looking at further, or ++ * else return false (zero) if it is not. ++ * ++ * This check -ignores- the distinction between various watermarks, ++ * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is ++ * found to be full for any variation of these watermarks, it will ++ * be considered full for up to one second by all requests, unless ++ * we are so low on memory on all allowed nodes that we are forced ++ * into the second scan of the zonelist. ++ * ++ * In the second scan we ignore this zonelist cache and exactly ++ * apply the watermarks to all zones, even it is slower to do so. ++ * We are low on memory in the second scan, and should leave no stone ++ * unturned looking for a free page. ++ */ ++static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z, ++ nodemask_t *allowednodes) ++{ ++ struct zonelist_cache *zlc; /* cached zonelist speedup info */ ++ int i; /* index of *z in zonelist zones */ ++ int n; /* node that zone *z is on */ ++ ++ zlc = zonelist->zlcache_ptr; ++ if (!zlc) ++ return 1; ++ ++ i = z - zonelist->_zonerefs; ++ n = zlc->z_to_n[i]; ++ ++ /* This zone is worth trying if it is allowed but not full */ ++ return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones); ++} ++ ++/* ++ * Given 'z' scanning a zonelist, set the corresponding bit in ++ * zlc->fullzones, so that subsequent attempts to allocate a page ++ * from that zone don't waste time re-examining it. ++ */ ++static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z) ++{ ++ struct zonelist_cache *zlc; /* cached zonelist speedup info */ ++ int i; /* index of *z in zonelist zones */ ++ ++ zlc = zonelist->zlcache_ptr; ++ if (!zlc) ++ return; ++ ++ i = z - zonelist->_zonerefs; ++ ++ set_bit(i, zlc->fullzones); ++} ++ ++/* ++ * clear all zones full, called after direct reclaim makes progress so that ++ * a zone that was recently full is not skipped over for up to a second ++ */ ++static void zlc_clear_zones_full(struct zonelist *zonelist) ++{ ++ struct zonelist_cache *zlc; /* cached zonelist speedup info */ ++ ++ zlc = zonelist->zlcache_ptr; ++ if (!zlc) ++ return; ++ ++ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); ++} ++ ++static bool zone_local(struct zone *local_zone, struct zone *zone) ++{ ++ return local_zone->node == zone->node; ++} ++ ++static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) ++{ ++ return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) < ++ RECLAIM_DISTANCE; ++} ++ ++#else /* CONFIG_NUMA */ ++ ++static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) ++{ ++ return NULL; ++} ++ ++static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z, ++ nodemask_t *allowednodes) ++{ ++ return 1; ++} ++ ++static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z) ++{ ++} ++ ++static void zlc_clear_zones_full(struct zonelist *zonelist) ++{ ++} ++ ++static bool zone_local(struct zone *local_zone, struct zone *zone) ++{ ++ return true; ++} ++ ++static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) ++{ ++ return true; ++} ++ ++#endif /* CONFIG_NUMA */ ++ ++static void reset_alloc_batches(struct zone *preferred_zone) ++{ ++ struct zone *zone = preferred_zone->zone_pgdat->node_zones; ++ ++ do { ++ mod_zone_page_state(zone, NR_ALLOC_BATCH, ++ high_wmark_pages(zone) - low_wmark_pages(zone) - ++ atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); ++ clear_bit(ZONE_FAIR_DEPLETED, &zone->flags); ++ } while (zone++ != preferred_zone); ++} ++ ++/* ++ * get_page_from_freelist goes through the zonelist trying to allocate ++ * a page. ++ */ ++static struct page * ++get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, ++ const struct alloc_context *ac) ++{ ++ struct zonelist *zonelist = ac->zonelist; ++ struct zoneref *z; ++ struct page *page = NULL; ++ struct zone *zone; ++ nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */ ++ int zlc_active = 0; /* set if using zonelist_cache */ ++ int did_zlc_setup = 0; /* just call zlc_setup() one time */ ++ bool consider_zone_dirty = (alloc_flags & ALLOC_WMARK_LOW) && ++ (gfp_mask & __GFP_WRITE); ++ int nr_fair_skipped = 0; ++ bool zonelist_rescan; ++ ++zonelist_scan: ++ zonelist_rescan = false; ++ ++ /* ++ * Scan zonelist, looking for a zone with enough free. ++ * See also __cpuset_node_allowed() comment in kernel/cpuset.c. ++ */ ++ for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, ++ ac->nodemask) { ++ unsigned long mark; ++ ++ if (IS_ENABLED(CONFIG_NUMA) && zlc_active && ++ !zlc_zone_worth_trying(zonelist, z, allowednodes)) ++ continue; ++ if (cpusets_enabled() && ++ (alloc_flags & ALLOC_CPUSET) && ++ !cpuset_zone_allowed(zone, gfp_mask)) ++ continue; ++ /* ++ * Distribute pages in proportion to the individual ++ * zone size to ensure fair page aging. The zone a ++ * page was allocated in should have no effect on the ++ * time the page has in memory before being reclaimed. ++ */ ++ if (alloc_flags & ALLOC_FAIR) { ++ if (!zone_local(ac->preferred_zone, zone)) ++ break; ++ if (test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) { ++ nr_fair_skipped++; ++ continue; ++ } ++ } ++ /* ++ * When allocating a page cache page for writing, we ++ * want to get it from a zone that is within its dirty ++ * limit, such that no single zone holds more than its ++ * proportional share of globally allowed dirty pages. ++ * The dirty limits take into account the zone's ++ * lowmem reserves and high watermark so that kswapd ++ * should be able to balance it without having to ++ * write pages from its LRU list. ++ * ++ * This may look like it could increase pressure on ++ * lower zones by failing allocations in higher zones ++ * before they are full. But the pages that do spill ++ * over are limited as the lower zones are protected ++ * by this very same mechanism. It should not become ++ * a practical burden to them. ++ * ++ * XXX: For now, allow allocations to potentially ++ * exceed the per-zone dirty limit in the slowpath ++ * (ALLOC_WMARK_LOW unset) before going into reclaim, ++ * which is important when on a NUMA setup the allowed ++ * zones are together not big enough to reach the ++ * global limit. The proper fix for these situations ++ * will require awareness of zones in the ++ * dirty-throttling and the flusher threads. ++ */ ++ if (consider_zone_dirty && !zone_dirty_ok(zone)) ++ continue; ++ ++ mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; ++ if (!zone_watermark_ok(zone, order, mark, ++ ac->classzone_idx, alloc_flags)) { ++ int ret; ++ ++ /* Checked here to keep the fast path fast */ ++ BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); ++ if (alloc_flags & ALLOC_NO_WATERMARKS) ++ goto try_this_zone; ++ ++ if (IS_ENABLED(CONFIG_NUMA) && ++ !did_zlc_setup && nr_online_nodes > 1) { ++ /* ++ * we do zlc_setup if there are multiple nodes ++ * and before considering the first zone allowed ++ * by the cpuset. ++ */ ++ allowednodes = zlc_setup(zonelist, alloc_flags); ++ zlc_active = 1; ++ did_zlc_setup = 1; ++ } ++ ++ if (zone_reclaim_mode == 0 || ++ !zone_allows_reclaim(ac->preferred_zone, zone)) ++ goto this_zone_full; ++ ++ /* ++ * As we may have just activated ZLC, check if the first ++ * eligible zone has failed zone_reclaim recently. ++ */ ++ if (IS_ENABLED(CONFIG_NUMA) && zlc_active && ++ !zlc_zone_worth_trying(zonelist, z, allowednodes)) ++ continue; ++ ++ ret = zone_reclaim(zone, gfp_mask, order); ++ switch (ret) { ++ case ZONE_RECLAIM_NOSCAN: ++ /* did not scan */ ++ continue; ++ case ZONE_RECLAIM_FULL: ++ /* scanned but unreclaimable */ ++ continue; ++ default: ++ /* did we reclaim enough */ ++ if (zone_watermark_ok(zone, order, mark, ++ ac->classzone_idx, alloc_flags)) ++ goto try_this_zone; ++ ++ /* ++ * Failed to reclaim enough to meet watermark. ++ * Only mark the zone full if checking the min ++ * watermark or if we failed to reclaim just ++ * 1<<order pages or else the page allocator ++ * fastpath will prematurely mark zones full ++ * when the watermark is between the low and ++ * min watermarks. ++ */ ++ if (((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN) || ++ ret == ZONE_RECLAIM_SOME) ++ goto this_zone_full; ++ ++ continue; ++ } ++ } ++ ++try_this_zone: ++ page = buffered_rmqueue(ac->preferred_zone, zone, order, ++ gfp_mask, ac->migratetype); ++ if (page) { ++ if (prep_new_page(page, order, gfp_mask, alloc_flags)) ++ goto try_this_zone; ++ return page; ++ } ++this_zone_full: ++ if (IS_ENABLED(CONFIG_NUMA) && zlc_active) ++ zlc_mark_zone_full(zonelist, z); ++ } ++ ++ /* ++ * The first pass makes sure allocations are spread fairly within the ++ * local node. However, the local node might have free pages left ++ * after the fairness batches are exhausted, and remote zones haven't ++ * even been considered yet. Try once more without fairness, and ++ * include remote zones now, before entering the slowpath and waking ++ * kswapd: prefer spilling to a remote zone over swapping locally. ++ */ ++ if (alloc_flags & ALLOC_FAIR) { ++ alloc_flags &= ~ALLOC_FAIR; ++ if (nr_fair_skipped) { ++ zonelist_rescan = true; ++ reset_alloc_batches(ac->preferred_zone); ++ } ++ if (nr_online_nodes > 1) ++ zonelist_rescan = true; ++ } ++ ++ if (unlikely(IS_ENABLED(CONFIG_NUMA) && zlc_active)) { ++ /* Disable zlc cache for second zonelist scan */ ++ zlc_active = 0; ++ zonelist_rescan = true; ++ } ++ ++ if (zonelist_rescan) ++ goto zonelist_scan; ++ ++ return NULL; ++} ++ ++/* ++ * Large machines with many possible nodes should not always dump per-node ++ * meminfo in irq context. ++ */ ++static inline bool should_suppress_show_mem(void) ++{ ++ bool ret = false; ++ ++#if NODES_SHIFT > 8 ++ ret = in_interrupt(); ++#endif ++ return ret; ++} ++ ++static DEFINE_RATELIMIT_STATE(nopage_rs, ++ DEFAULT_RATELIMIT_INTERVAL, ++ DEFAULT_RATELIMIT_BURST); ++ ++void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...) ++{ ++ unsigned int filter = SHOW_MEM_FILTER_NODES; ++ ++ if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) || ++ debug_guardpage_minorder() > 0) ++ return; ++ ++ /* ++ * This documents exceptions given to allocations in certain ++ * contexts that are allowed to allocate outside current's set ++ * of allowed nodes. ++ */ ++ if (!(gfp_mask & __GFP_NOMEMALLOC)) ++ if (test_thread_flag(TIF_MEMDIE) || ++ (current->flags & (PF_MEMALLOC | PF_EXITING))) ++ filter &= ~SHOW_MEM_FILTER_NODES; ++ if (in_interrupt() || !(gfp_mask & __GFP_WAIT)) ++ filter &= ~SHOW_MEM_FILTER_NODES; ++ ++ if (fmt) { ++ struct va_format vaf; ++ va_list args; ++ ++ va_start(args, fmt); ++ ++ vaf.fmt = fmt; ++ vaf.va = &args; ++ ++ pr_warn("%pV", &vaf); ++ ++ va_end(args); ++ } ++ ++ pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n", ++ current->comm, order, gfp_mask); ++ ++ dump_stack(); ++ if (!should_suppress_show_mem()) ++ show_mem(filter); ++} ++ ++static inline int ++should_alloc_retry(gfp_t gfp_mask, unsigned int order, ++ unsigned long did_some_progress, ++ unsigned long pages_reclaimed) ++{ ++ /* Do not loop if specifically requested */ ++ if (gfp_mask & __GFP_NORETRY) ++ return 0; ++ ++ /* Always retry if specifically requested */ ++ if (gfp_mask & __GFP_NOFAIL) ++ return 1; ++ ++ /* ++ * Suspend converts GFP_KERNEL to __GFP_WAIT which can prevent reclaim ++ * making forward progress without invoking OOM. Suspend also disables ++ * storage devices so kswapd will not help. Bail if we are suspending. ++ */ ++ if (!did_some_progress && pm_suspended_storage()) ++ return 0; ++ ++ /* ++ * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER ++ * means __GFP_NOFAIL, but that may not be true in other ++ * implementations. ++ */ ++ if (order <= PAGE_ALLOC_COSTLY_ORDER) ++ return 1; ++ ++ /* ++ * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is ++ * specified, then we retry until we no longer reclaim any pages ++ * (above), or we've reclaimed an order of pages at least as ++ * large as the allocation's order. In both cases, if the ++ * allocation still fails, we stop retrying. ++ */ ++ if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order)) ++ return 1; ++ ++ return 0; ++} ++ ++static inline struct page * ++__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, ++ const struct alloc_context *ac, unsigned long *did_some_progress) ++{ ++ struct page *page; ++ ++ *did_some_progress = 0; ++ ++ /* ++ * Acquire the per-zone oom lock for each zone. If that ++ * fails, somebody else is making progress for us. ++ */ ++ if (!oom_zonelist_trylock(ac->zonelist, gfp_mask)) { ++ *did_some_progress = 1; ++ schedule_timeout_uninterruptible(1); ++ return NULL; ++ } ++ ++ /* ++ * Go through the zonelist yet one more time, keep very high watermark ++ * here, this is only to catch a parallel oom killing, we must fail if ++ * we're still under heavy pressure. ++ */ ++ page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order, ++ ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); ++ if (page) ++ goto out; ++ ++ if (!(gfp_mask & __GFP_NOFAIL)) { ++ /* Coredumps can quickly deplete all memory reserves */ ++ if (current->flags & PF_DUMPCORE) ++ goto out; ++ /* The OOM killer will not help higher order allocs */ ++ if (order > PAGE_ALLOC_COSTLY_ORDER) ++ goto out; ++ /* The OOM killer does not needlessly kill tasks for lowmem */ ++ if (ac->high_zoneidx < ZONE_NORMAL) ++ goto out; ++ /* The OOM killer does not compensate for light reclaim */ ++ if (!(gfp_mask & __GFP_FS)) { ++ /* ++ * XXX: Page reclaim didn't yield anything, ++ * and the OOM killer can't be invoked, but ++ * keep looping as per should_alloc_retry(). ++ */ ++ *did_some_progress = 1; ++ goto out; ++ } ++ /* The OOM killer may not free memory on a specific node */ ++ if (gfp_mask & __GFP_THISNODE) ++ goto out; ++ } ++ /* Exhausted what can be done so it's blamo time */ ++ if (out_of_memory(ac->zonelist, gfp_mask, order, ac->nodemask, false) ++ || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) ++ *did_some_progress = 1; ++out: ++ oom_zonelist_unlock(ac->zonelist, gfp_mask); ++ return page; ++} ++ ++#ifdef CONFIG_COMPACTION ++/* Try memory compaction for high-order allocations before reclaim */ ++static struct page * ++__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, ++ int alloc_flags, const struct alloc_context *ac, ++ enum migrate_mode mode, int *contended_compaction, ++ bool *deferred_compaction) ++{ ++ unsigned long compact_result; ++ struct page *page; ++ ++ if (!order) ++ return NULL; ++ ++ current->flags |= PF_MEMALLOC; ++ compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, ++ mode, contended_compaction); ++ current->flags &= ~PF_MEMALLOC; ++ ++ switch (compact_result) { ++ case COMPACT_DEFERRED: ++ *deferred_compaction = true; ++ /* fall-through */ ++ case COMPACT_SKIPPED: ++ return NULL; ++ default: ++ break; ++ } ++ ++ /* ++ * At least in one zone compaction wasn't deferred or skipped, so let's ++ * count a compaction stall ++ */ ++ count_vm_event(COMPACTSTALL); ++ ++ page = get_page_from_freelist(gfp_mask, order, ++ alloc_flags & ~ALLOC_NO_WATERMARKS, ac); ++ ++ if (page) { ++ struct zone *zone = page_zone(page); ++ ++ zone->compact_blockskip_flush = false; ++ compaction_defer_reset(zone, order, true); ++ count_vm_event(COMPACTSUCCESS); ++ return page; ++ } ++ ++ /* ++ * It's bad if compaction run occurs and fails. The most likely reason ++ * is that pages exist, but not enough to satisfy watermarks. ++ */ ++ count_vm_event(COMPACTFAIL); ++ ++ cond_resched(); ++ ++ return NULL; ++} ++#else ++static inline struct page * ++__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, ++ int alloc_flags, const struct alloc_context *ac, ++ enum migrate_mode mode, int *contended_compaction, ++ bool *deferred_compaction) ++{ ++ return NULL; ++} ++#endif /* CONFIG_COMPACTION */ ++ ++/* Perform direct synchronous page reclaim */ ++static int ++__perform_reclaim(gfp_t gfp_mask, unsigned int order, ++ const struct alloc_context *ac) ++{ ++ struct reclaim_state reclaim_state; ++ int progress; ++ ++ cond_resched(); ++ ++ /* We now go into synchronous reclaim */ ++ cpuset_memory_pressure_bump(); ++ current->flags |= PF_MEMALLOC; ++ lockdep_set_current_reclaim_state(gfp_mask); ++ reclaim_state.reclaimed_slab = 0; ++ current->reclaim_state = &reclaim_state; ++ ++ progress = try_to_free_pages(ac->zonelist, order, gfp_mask, ++ ac->nodemask); ++ ++ current->reclaim_state = NULL; ++ lockdep_clear_current_reclaim_state(); ++ current->flags &= ~PF_MEMALLOC; ++ ++ cond_resched(); ++ ++ return progress; ++} ++ ++/* The really slow allocator path where we enter direct reclaim */ ++static inline struct page * ++__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, ++ int alloc_flags, const struct alloc_context *ac, ++ unsigned long *did_some_progress) ++{ ++ struct page *page = NULL; ++ bool drained = false; ++ ++ *did_some_progress = __perform_reclaim(gfp_mask, order, ac); ++ if (unlikely(!(*did_some_progress))) ++ return NULL; ++ ++ /* After successful reclaim, reconsider all zones for allocation */ ++ if (IS_ENABLED(CONFIG_NUMA)) ++ zlc_clear_zones_full(ac->zonelist); ++ ++retry: ++ page = get_page_from_freelist(gfp_mask, order, ++ alloc_flags & ~ALLOC_NO_WATERMARKS, ac); ++ ++ /* ++ * If an allocation failed after direct reclaim, it could be because ++ * pages are pinned on the per-cpu lists. Drain them and try again ++ */ ++ if (!page && !drained) { ++ drain_all_pages(NULL); ++ drained = true; ++ goto retry; ++ } ++ ++ return page; ++} ++ ++/* ++ * This is called in the allocator slow-path if the allocation request is of ++ * sufficient urgency to ignore watermarks and take other desperate measures ++ */ ++static inline struct page * ++__alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order, ++ const struct alloc_context *ac) ++{ ++ struct page *page; ++ ++ do { ++ page = get_page_from_freelist(gfp_mask, order, ++ ALLOC_NO_WATERMARKS, ac); ++ ++ if (!page && gfp_mask & __GFP_NOFAIL) ++ wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, ++ HZ/50); ++ } while (!page && (gfp_mask & __GFP_NOFAIL)); ++ ++ return page; ++} ++ ++static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac) ++{ ++ struct zoneref *z; ++ struct zone *zone; ++ ++ for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ++ ac->high_zoneidx, ac->nodemask) ++ wakeup_kswapd(zone, order, zone_idx(ac->preferred_zone)); ++} ++ ++static inline int ++gfp_to_alloc_flags(gfp_t gfp_mask) ++{ ++ int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; ++ const bool atomic = !(gfp_mask & (__GFP_WAIT | __GFP_NO_KSWAPD)); ++ ++ /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ ++ BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); ++ ++ /* ++ * The caller may dip into page reserves a bit more if the caller ++ * cannot run direct reclaim, or if the caller has realtime scheduling ++ * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will ++ * set both ALLOC_HARDER (atomic == true) and ALLOC_HIGH (__GFP_HIGH). ++ */ ++ alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); ++ ++ if (atomic) { ++ /* ++ * Not worth trying to allocate harder for __GFP_NOMEMALLOC even ++ * if it can't schedule. ++ */ ++ if (!(gfp_mask & __GFP_NOMEMALLOC)) ++ alloc_flags |= ALLOC_HARDER; ++ /* ++ * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the ++ * comment for __cpuset_node_allowed(). ++ */ ++ alloc_flags &= ~ALLOC_CPUSET; ++ } else if (unlikely(rt_task(current)) && !in_interrupt()) ++ alloc_flags |= ALLOC_HARDER; ++ ++ if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) { ++ if (gfp_mask & __GFP_MEMALLOC) ++ alloc_flags |= ALLOC_NO_WATERMARKS; ++ else if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) ++ alloc_flags |= ALLOC_NO_WATERMARKS; ++ else if (!in_interrupt() && ++ ((current->flags & PF_MEMALLOC) || ++ unlikely(test_thread_flag(TIF_MEMDIE)))) ++ alloc_flags |= ALLOC_NO_WATERMARKS; ++ } ++#ifdef CONFIG_CMA ++ if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) ++ alloc_flags |= ALLOC_CMA; ++#endif ++ return alloc_flags; ++} ++ ++bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) ++{ ++ return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS); ++} ++ ++static inline struct page * ++__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, ++ struct alloc_context *ac) ++{ ++ const gfp_t wait = gfp_mask & __GFP_WAIT; ++ struct page *page = NULL; ++ int alloc_flags; ++ unsigned long pages_reclaimed = 0; ++ unsigned long did_some_progress; ++ enum migrate_mode migration_mode = MIGRATE_ASYNC; ++ bool deferred_compaction = false; ++ int contended_compaction = COMPACT_CONTENDED_NONE; ++ ++ /* ++ * In the slowpath, we sanity check order to avoid ever trying to ++ * reclaim >= MAX_ORDER areas which will never succeed. Callers may ++ * be using allocators in order of preference for an area that is ++ * too large. ++ */ ++ if (order >= MAX_ORDER) { ++ WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); ++ return NULL; ++ } ++ ++ /* ++ * If this allocation cannot block and it is for a specific node, then ++ * fail early. There's no need to wakeup kswapd or retry for a ++ * speculative node-specific allocation. ++ */ ++ if (IS_ENABLED(CONFIG_NUMA) && (gfp_mask & __GFP_THISNODE) && !wait) ++ goto nopage; ++ ++retry: ++ if (!(gfp_mask & __GFP_NO_KSWAPD)) ++ wake_all_kswapds(order, ac); ++ ++ /* ++ * OK, we're below the kswapd watermark and have kicked background ++ * reclaim. Now things get more complex, so set up alloc_flags according ++ * to how we want to proceed. ++ */ ++ alloc_flags = gfp_to_alloc_flags(gfp_mask); ++ ++ /* ++ * Find the true preferred zone if the allocation is unconstrained by ++ * cpusets. ++ */ ++ if (!(alloc_flags & ALLOC_CPUSET) && !ac->nodemask) { ++ struct zoneref *preferred_zoneref; ++ preferred_zoneref = first_zones_zonelist(ac->zonelist, ++ ac->high_zoneidx, NULL, &ac->preferred_zone); ++ ac->classzone_idx = zonelist_zone_idx(preferred_zoneref); ++ } ++ ++ /* This is the last chance, in general, before the goto nopage. */ ++ page = get_page_from_freelist(gfp_mask, order, ++ alloc_flags & ~ALLOC_NO_WATERMARKS, ac); ++ if (page) ++ goto got_pg; ++ ++ /* Allocate without watermarks if the context allows */ ++ if (alloc_flags & ALLOC_NO_WATERMARKS) { ++ /* ++ * Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds ++ * the allocation is high priority and these type of ++ * allocations are system rather than user orientated ++ */ ++ ac->zonelist = node_zonelist(numa_node_id(), gfp_mask); ++ ++ page = __alloc_pages_high_priority(gfp_mask, order, ac); ++ ++ if (page) { ++ goto got_pg; ++ } ++ } ++ ++ /* Atomic allocations - we can't balance anything */ ++ if (!wait) { ++ /* ++ * All existing users of the deprecated __GFP_NOFAIL are ++ * blockable, so warn of any new users that actually allow this ++ * type of allocation to fail. ++ */ ++ WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL); ++ goto nopage; ++ } ++ ++ /* Avoid recursion of direct reclaim */ ++ if (current->flags & PF_MEMALLOC) ++ goto nopage; ++ ++ /* Avoid allocations with no watermarks from looping endlessly */ ++ if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL)) ++ goto nopage; ++ ++ /* ++ * Try direct compaction. The first pass is asynchronous. Subsequent ++ * attempts after direct reclaim are synchronous ++ */ ++ page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, ++ migration_mode, ++ &contended_compaction, ++ &deferred_compaction); ++ if (page) ++ goto got_pg; ++ ++ /* Checks for THP-specific high-order allocations */ ++ if ((gfp_mask & GFP_TRANSHUGE) == GFP_TRANSHUGE) { ++ /* ++ * If compaction is deferred for high-order allocations, it is ++ * because sync compaction recently failed. If this is the case ++ * and the caller requested a THP allocation, we do not want ++ * to heavily disrupt the system, so we fail the allocation ++ * instead of entering direct reclaim. ++ */ ++ if (deferred_compaction) ++ goto nopage; ++ ++ /* ++ * In all zones where compaction was attempted (and not ++ * deferred or skipped), lock contention has been detected. ++ * For THP allocation we do not want to disrupt the others ++ * so we fallback to base pages instead. ++ */ ++ if (contended_compaction == COMPACT_CONTENDED_LOCK) ++ goto nopage; ++ ++ /* ++ * If compaction was aborted due to need_resched(), we do not ++ * want to further increase allocation latency, unless it is ++ * khugepaged trying to collapse. ++ */ ++ if (contended_compaction == COMPACT_CONTENDED_SCHED ++ && !(current->flags & PF_KTHREAD)) ++ goto nopage; ++ } ++ ++ /* ++ * It can become very expensive to allocate transparent hugepages at ++ * fault, so use asynchronous memory compaction for THP unless it is ++ * khugepaged trying to collapse. ++ */ ++ if ((gfp_mask & GFP_TRANSHUGE) != GFP_TRANSHUGE || ++ (current->flags & PF_KTHREAD)) ++ migration_mode = MIGRATE_SYNC_LIGHT; ++ ++ /* Try direct reclaim and then allocating */ ++ page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, ++ &did_some_progress); ++ if (page) ++ goto got_pg; ++ ++ /* Check if we should retry the allocation */ ++ pages_reclaimed += did_some_progress; ++ if (should_alloc_retry(gfp_mask, order, did_some_progress, ++ pages_reclaimed)) { ++ /* ++ * If we fail to make progress by freeing individual ++ * pages, but the allocation wants us to keep going, ++ * start OOM killing tasks. ++ */ ++ if (!did_some_progress) { ++ page = __alloc_pages_may_oom(gfp_mask, order, ac, ++ &did_some_progress); ++ if (page) ++ goto got_pg; ++ if (!did_some_progress) ++ goto nopage; ++ } ++ /* Wait for some write requests to complete then retry */ ++ wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, HZ/50); ++ goto retry; ++ } else { ++ /* ++ * High-order allocations do not necessarily loop after ++ * direct reclaim and reclaim/compaction depends on compaction ++ * being called after reclaim so call directly if necessary ++ */ ++ page = __alloc_pages_direct_compact(gfp_mask, order, ++ alloc_flags, ac, migration_mode, ++ &contended_compaction, ++ &deferred_compaction); ++ if (page) ++ goto got_pg; ++ } ++ ++nopage: ++ warn_alloc_failed(gfp_mask, order, NULL); ++got_pg: ++ return page; ++} ++ ++/* ++ * This is the 'heart' of the zoned buddy allocator. ++ */ ++struct page * ++__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, ++ struct zonelist *zonelist, nodemask_t *nodemask) ++{ ++ struct zoneref *preferred_zoneref; ++ struct page *page = NULL; ++ unsigned int cpuset_mems_cookie; ++ int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR; ++ gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ ++ struct alloc_context ac = { ++ .high_zoneidx = gfp_zone(gfp_mask), ++ .nodemask = nodemask, ++ .migratetype = gfpflags_to_migratetype(gfp_mask), ++ }; ++ ++ gfp_mask &= gfp_allowed_mask; ++ ++ lockdep_trace_alloc(gfp_mask); ++ ++ might_sleep_if(gfp_mask & __GFP_WAIT); ++ ++ if (should_fail_alloc_page(gfp_mask, order)) ++ return NULL; ++ ++ /* ++ * Check the zones suitable for the gfp_mask contain at least one ++ * valid zone. It's possible to have an empty zonelist as a result ++ * of __GFP_THISNODE and a memoryless node ++ */ ++ if (unlikely(!zonelist->_zonerefs->zone)) ++ return NULL; ++ ++ if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE) ++ alloc_flags |= ALLOC_CMA; ++ ++retry_cpuset: ++ cpuset_mems_cookie = read_mems_allowed_begin(); ++ ++ /* We set it here, as __alloc_pages_slowpath might have changed it */ ++ ac.zonelist = zonelist; ++ /* The preferred zone is used for statistics later */ ++ preferred_zoneref = first_zones_zonelist(ac.zonelist, ac.high_zoneidx, ++ ac.nodemask ? : &cpuset_current_mems_allowed, ++ &ac.preferred_zone); ++ if (!ac.preferred_zone) ++ goto out; ++ ac.classzone_idx = zonelist_zone_idx(preferred_zoneref); ++ ++ /* First allocation attempt */ ++ alloc_mask = gfp_mask|__GFP_HARDWALL; ++ page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); ++ if (unlikely(!page)) { ++ /* ++ * Runtime PM, block IO and its error handling path ++ * can deadlock because I/O on the device might not ++ * complete. ++ */ ++ alloc_mask = memalloc_noio_flags(gfp_mask); ++ ++ page = __alloc_pages_slowpath(alloc_mask, order, &ac); ++ } ++ ++ if (kmemcheck_enabled && page) ++ kmemcheck_pagealloc_alloc(page, order, gfp_mask); ++ ++ trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); ++ ++out: ++ /* ++ * When updating a task's mems_allowed, it is possible to race with ++ * parallel threads in such a way that an allocation can fail while ++ * the mask is being updated. If a page allocation is about to fail, ++ * check if the cpuset changed during allocation and if so, retry. ++ */ ++ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) ++ goto retry_cpuset; ++ ++ return page; ++} ++EXPORT_SYMBOL(__alloc_pages_nodemask); ++ ++/* ++ * Common helper functions. ++ */ ++unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) ++{ ++ struct page *page; ++ ++ /* ++ * __get_free_pages() returns a 32-bit address, which cannot represent ++ * a highmem page ++ */ ++ VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0); ++ ++ page = alloc_pages(gfp_mask, order); ++ if (!page) ++ return 0; ++ return (unsigned long) page_address(page); ++} ++EXPORT_SYMBOL(__get_free_pages); ++ ++unsigned long get_zeroed_page(gfp_t gfp_mask) ++{ ++ return __get_free_pages(gfp_mask | __GFP_ZERO, 0); ++} ++EXPORT_SYMBOL(get_zeroed_page); ++ ++void __free_pages(struct page *page, unsigned int order) ++{ ++ if (put_page_testzero(page)) { ++ if (order == 0) ++ free_hot_cold_page(page, false); ++ else ++ __free_pages_ok(page, order); ++ } ++} ++ ++EXPORT_SYMBOL(__free_pages); ++ ++void free_pages(unsigned long addr, unsigned int order) ++{ ++ if (addr != 0) { ++ VM_BUG_ON(!virt_addr_valid((void *)addr)); ++ __free_pages(virt_to_page((void *)addr), order); ++ } ++} ++ ++EXPORT_SYMBOL(free_pages); ++ ++/* ++ * alloc_kmem_pages charges newly allocated pages to the kmem resource counter ++ * of the current memory cgroup. ++ * ++ * It should be used when the caller would like to use kmalloc, but since the ++ * allocation is large, it has to fall back to the page allocator. ++ */ ++struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order) ++{ ++ struct page *page; ++ struct mem_cgroup *memcg = NULL; ++ ++ if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order)) ++ return NULL; ++ page = alloc_pages(gfp_mask, order); ++ memcg_kmem_commit_charge(page, memcg, order); ++ return page; ++} ++ ++struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask, unsigned int order) ++{ ++ struct page *page; ++ struct mem_cgroup *memcg = NULL; ++ ++ if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order)) ++ return NULL; ++ page = alloc_pages_node(nid, gfp_mask, order); ++ memcg_kmem_commit_charge(page, memcg, order); ++ return page; ++} ++ ++/* ++ * __free_kmem_pages and free_kmem_pages will free pages allocated with ++ * alloc_kmem_pages. ++ */ ++void __free_kmem_pages(struct page *page, unsigned int order) ++{ ++ memcg_kmem_uncharge_pages(page, order); ++ __free_pages(page, order); ++} ++ ++void free_kmem_pages(unsigned long addr, unsigned int order) ++{ ++ if (addr != 0) { ++ VM_BUG_ON(!virt_addr_valid((void *)addr)); ++ __free_kmem_pages(virt_to_page((void *)addr), order); ++ } ++} ++ ++static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size) ++{ ++ if (addr) { ++ unsigned long alloc_end = addr + (PAGE_SIZE << order); ++ unsigned long used = addr + PAGE_ALIGN(size); ++ ++ split_page(virt_to_page((void *)addr), order); ++ while (used < alloc_end) { ++ free_page(used); ++ used += PAGE_SIZE; ++ } ++ } ++ return (void *)addr; ++} ++ ++/** ++ * alloc_pages_exact - allocate an exact number physically-contiguous pages. ++ * @size: the number of bytes to allocate ++ * @gfp_mask: GFP flags for the allocation ++ * ++ * This function is similar to alloc_pages(), except that it allocates the ++ * minimum number of pages to satisfy the request. alloc_pages() can only ++ * allocate memory in power-of-two pages. ++ * ++ * This function is also limited by MAX_ORDER. ++ * ++ * Memory allocated by this function must be released by free_pages_exact(). ++ */ ++void *alloc_pages_exact(size_t size, gfp_t gfp_mask) ++{ ++ unsigned int order = get_order(size); ++ unsigned long addr; ++ ++ addr = __get_free_pages(gfp_mask, order); ++ return make_alloc_exact(addr, order, size); ++} ++EXPORT_SYMBOL(alloc_pages_exact); ++ ++/** ++ * alloc_pages_exact_nid - allocate an exact number of physically-contiguous ++ * pages on a node. ++ * @nid: the preferred node ID where memory should be allocated ++ * @size: the number of bytes to allocate ++ * @gfp_mask: GFP flags for the allocation ++ * ++ * Like alloc_pages_exact(), but try to allocate on node nid first before falling ++ * back. ++ * Note this is not alloc_pages_exact_node() which allocates on a specific node, ++ * but is not exact. ++ */ ++void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) ++{ ++ unsigned order = get_order(size); ++ struct page *p = alloc_pages_node(nid, gfp_mask, order); ++ if (!p) ++ return NULL; ++ return make_alloc_exact((unsigned long)page_address(p), order, size); ++} ++ ++/** ++ * free_pages_exact - release memory allocated via alloc_pages_exact() ++ * @virt: the value returned by alloc_pages_exact. ++ * @size: size of allocation, same value as passed to alloc_pages_exact(). ++ * ++ * Release the memory allocated by a previous call to alloc_pages_exact. ++ */ ++void free_pages_exact(void *virt, size_t size) ++{ ++ unsigned long addr = (unsigned long)virt; ++ unsigned long end = addr + PAGE_ALIGN(size); ++ ++ while (addr < end) { ++ free_page(addr); ++ addr += PAGE_SIZE; ++ } ++} ++EXPORT_SYMBOL(free_pages_exact); ++ ++/** ++ * nr_free_zone_pages - count number of pages beyond high watermark ++ * @offset: The zone index of the highest zone ++ * ++ * nr_free_zone_pages() counts the number of counts pages which are beyond the ++ * high watermark within all zones at or below a given zone index. For each ++ * zone, the number of pages is calculated as: ++ * managed_pages - high_pages ++ */ ++static unsigned long nr_free_zone_pages(int offset) ++{ ++ struct zoneref *z; ++ struct zone *zone; ++ ++ /* Just pick one node, since fallback list is circular */ ++ unsigned long sum = 0; ++ ++ struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); ++ ++ for_each_zone_zonelist(zone, z, zonelist, offset) { ++ unsigned long size = zone->managed_pages; ++ unsigned long high = high_wmark_pages(zone); ++ if (size > high) ++ sum += size - high; ++ } ++ ++ return sum; ++} ++ ++/** ++ * nr_free_buffer_pages - count number of pages beyond high watermark ++ * ++ * nr_free_buffer_pages() counts the number of pages which are beyond the high ++ * watermark within ZONE_DMA and ZONE_NORMAL. ++ */ ++unsigned long nr_free_buffer_pages(void) ++{ ++ return nr_free_zone_pages(gfp_zone(GFP_USER)); ++} ++EXPORT_SYMBOL_GPL(nr_free_buffer_pages); ++ ++/** ++ * nr_free_pagecache_pages - count number of pages beyond high watermark ++ * ++ * nr_free_pagecache_pages() counts the number of pages which are beyond the ++ * high watermark within all zones. ++ */ ++unsigned long nr_free_pagecache_pages(void) ++{ ++ return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); ++} ++ ++static inline void show_node(struct zone *zone) ++{ ++ if (IS_ENABLED(CONFIG_NUMA)) ++ printk("Node %d ", zone_to_nid(zone)); ++} ++ ++void si_meminfo(struct sysinfo *val) ++{ ++ val->totalram = totalram_pages; ++ val->sharedram = global_page_state(NR_SHMEM); ++ val->freeram = global_page_state(NR_FREE_PAGES); ++ val->bufferram = nr_blockdev_pages(); ++ val->totalhigh = totalhigh_pages; ++ val->freehigh = nr_free_highpages(); ++ val->mem_unit = PAGE_SIZE; ++} ++ ++EXPORT_SYMBOL(si_meminfo); ++ ++#ifdef CONFIG_NUMA ++void si_meminfo_node(struct sysinfo *val, int nid) ++{ ++ int zone_type; /* needs to be signed */ ++ unsigned long managed_pages = 0; ++ pg_data_t *pgdat = NODE_DATA(nid); ++ ++ for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) ++ managed_pages += pgdat->node_zones[zone_type].managed_pages; ++ val->totalram = managed_pages; ++ val->sharedram = node_page_state(nid, NR_SHMEM); ++ val->freeram = node_page_state(nid, NR_FREE_PAGES); ++#ifdef CONFIG_HIGHMEM ++ val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].managed_pages; ++ val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM], ++ NR_FREE_PAGES); ++#else ++ val->totalhigh = 0; ++ val->freehigh = 0; ++#endif ++ val->mem_unit = PAGE_SIZE; ++} ++#endif ++ ++/* ++ * Determine whether the node should be displayed or not, depending on whether ++ * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). ++ */ ++bool skip_free_areas_node(unsigned int flags, int nid) ++{ ++ bool ret = false; ++ unsigned int cpuset_mems_cookie; ++ ++ if (!(flags & SHOW_MEM_FILTER_NODES)) ++ goto out; ++ ++ do { ++ cpuset_mems_cookie = read_mems_allowed_begin(); ++ ret = !node_isset(nid, cpuset_current_mems_allowed); ++ } while (read_mems_allowed_retry(cpuset_mems_cookie)); ++out: ++ return ret; ++} ++ ++#define K(x) ((x) << (PAGE_SHIFT-10)) ++ ++static void show_migration_types(unsigned char type) ++{ ++ static const char types[MIGRATE_TYPES] = { ++ [MIGRATE_UNMOVABLE] = 'U', ++ [MIGRATE_RECLAIMABLE] = 'E', ++ [MIGRATE_MOVABLE] = 'M', ++ [MIGRATE_RESERVE] = 'R', ++#ifdef CONFIG_CMA ++ [MIGRATE_CMA] = 'C', ++#endif ++#ifdef CONFIG_MEMORY_ISOLATION ++ [MIGRATE_ISOLATE] = 'I', ++#endif ++ }; ++ char tmp[MIGRATE_TYPES + 1]; ++ char *p = tmp; ++ int i; ++ ++ for (i = 0; i < MIGRATE_TYPES; i++) { ++ if (type & (1 << i)) ++ *p++ = types[i]; ++ } ++ ++ *p = '\0'; ++ printk("(%s) ", tmp); ++} ++ ++/* ++ * Show free area list (used inside shift_scroll-lock stuff) ++ * We also calculate the percentage fragmentation. We do this by counting the ++ * memory on each free list with the exception of the first item on the list. ++ * ++ * Bits in @filter: ++ * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's ++ * cpuset. ++ */ ++void show_free_areas(unsigned int filter) ++{ ++ unsigned long free_pcp = 0; ++ int cpu; ++ struct zone *zone; ++ ++ for_each_populated_zone(zone) { ++ if (skip_free_areas_node(filter, zone_to_nid(zone))) ++ continue; ++ ++ for_each_online_cpu(cpu) ++ free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; ++ } ++ ++ printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" ++ " active_file:%lu inactive_file:%lu isolated_file:%lu\n" ++ " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" ++ " slab_reclaimable:%lu slab_unreclaimable:%lu\n" ++ " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" ++ " free:%lu free_pcp:%lu free_cma:%lu\n", ++ global_page_state(NR_ACTIVE_ANON), ++ global_page_state(NR_INACTIVE_ANON), ++ global_page_state(NR_ISOLATED_ANON), ++ global_page_state(NR_ACTIVE_FILE), ++ global_page_state(NR_INACTIVE_FILE), ++ global_page_state(NR_ISOLATED_FILE), ++ global_page_state(NR_UNEVICTABLE), ++ global_page_state(NR_FILE_DIRTY), ++ global_page_state(NR_WRITEBACK), ++ global_page_state(NR_UNSTABLE_NFS), ++ global_page_state(NR_SLAB_RECLAIMABLE), ++ global_page_state(NR_SLAB_UNRECLAIMABLE), ++ global_page_state(NR_FILE_MAPPED), ++ global_page_state(NR_SHMEM), ++ global_page_state(NR_PAGETABLE), ++ global_page_state(NR_BOUNCE), ++ global_page_state(NR_FREE_PAGES), ++ free_pcp, ++ global_page_state(NR_FREE_CMA_PAGES)); ++ ++ for_each_populated_zone(zone) { ++ int i; ++ ++ if (skip_free_areas_node(filter, zone_to_nid(zone))) ++ continue; ++ ++ free_pcp = 0; ++ for_each_online_cpu(cpu) ++ free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; ++ ++ show_node(zone); ++ printk("%s" ++ " free:%lukB" ++ " min:%lukB" ++ " low:%lukB" ++ " high:%lukB" ++ " active_anon:%lukB" ++ " inactive_anon:%lukB" ++ " active_file:%lukB" ++ " inactive_file:%lukB" ++ " unevictable:%lukB" ++ " isolated(anon):%lukB" ++ " isolated(file):%lukB" ++ " present:%lukB" ++ " managed:%lukB" ++ " mlocked:%lukB" ++ " dirty:%lukB" ++ " writeback:%lukB" ++ " mapped:%lukB" ++ " shmem:%lukB" ++ " slab_reclaimable:%lukB" ++ " slab_unreclaimable:%lukB" ++ " kernel_stack:%lukB" ++ " pagetables:%lukB" ++ " unstable:%lukB" ++ " bounce:%lukB" ++ " free_pcp:%lukB" ++ " local_pcp:%ukB" ++ " free_cma:%lukB" ++ " writeback_tmp:%lukB" ++ " pages_scanned:%lu" ++ " all_unreclaimable? %s" ++ "\n", ++ zone->name, ++ K(zone_page_state(zone, NR_FREE_PAGES)), ++ K(min_wmark_pages(zone)), ++ K(low_wmark_pages(zone)), ++ K(high_wmark_pages(zone)), ++ K(zone_page_state(zone, NR_ACTIVE_ANON)), ++ K(zone_page_state(zone, NR_INACTIVE_ANON)), ++ K(zone_page_state(zone, NR_ACTIVE_FILE)), ++ K(zone_page_state(zone, NR_INACTIVE_FILE)), ++ K(zone_page_state(zone, NR_UNEVICTABLE)), ++ K(zone_page_state(zone, NR_ISOLATED_ANON)), ++ K(zone_page_state(zone, NR_ISOLATED_FILE)), ++ K(zone->present_pages), ++ K(zone->managed_pages), ++ K(zone_page_state(zone, NR_MLOCK)), ++ K(zone_page_state(zone, NR_FILE_DIRTY)), ++ K(zone_page_state(zone, NR_WRITEBACK)), ++ K(zone_page_state(zone, NR_FILE_MAPPED)), ++ K(zone_page_state(zone, NR_SHMEM)), ++ K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)), ++ K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)), ++ zone_page_state(zone, NR_KERNEL_STACK) * ++ THREAD_SIZE / 1024, ++ K(zone_page_state(zone, NR_PAGETABLE)), ++ K(zone_page_state(zone, NR_UNSTABLE_NFS)), ++ K(zone_page_state(zone, NR_BOUNCE)), ++ K(free_pcp), ++ K(this_cpu_read(zone->pageset->pcp.count)), ++ K(zone_page_state(zone, NR_FREE_CMA_PAGES)), ++ K(zone_page_state(zone, NR_WRITEBACK_TEMP)), ++ K(zone_page_state(zone, NR_PAGES_SCANNED)), ++ (!zone_reclaimable(zone) ? "yes" : "no") ++ ); ++ printk("lowmem_reserve[]:"); ++ for (i = 0; i < MAX_NR_ZONES; i++) ++ printk(" %ld", zone->lowmem_reserve[i]); ++ printk("\n"); ++ } ++ ++ for_each_populated_zone(zone) { ++ unsigned long nr[MAX_ORDER], flags, order, total = 0; ++ unsigned char types[MAX_ORDER]; ++ ++ if (skip_free_areas_node(filter, zone_to_nid(zone))) ++ continue; ++ show_node(zone); ++ printk("%s: ", zone->name); ++ ++ spin_lock_irqsave(&zone->lock, flags); ++ for (order = 0; order < MAX_ORDER; order++) { ++ struct free_area *area = &zone->free_area[order]; ++ int type; ++ ++ nr[order] = area->nr_free; ++ total += nr[order] << order; ++ ++ types[order] = 0; ++ for (type = 0; type < MIGRATE_TYPES; type++) { ++ if (!list_empty(&area->free_list[type])) ++ types[order] |= 1 << type; ++ } ++ } ++ spin_unlock_irqrestore(&zone->lock, flags); ++ for (order = 0; order < MAX_ORDER; order++) { ++ printk("%lu*%lukB ", nr[order], K(1UL) << order); ++ if (nr[order]) ++ show_migration_types(types[order]); ++ } ++ printk("= %lukB\n", K(total)); ++ } ++ ++ hugetlb_show_meminfo(); ++ ++ printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES)); ++ ++ show_swap_cache_info(); ++} ++ ++static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) ++{ ++ zoneref->zone = zone; ++ zoneref->zone_idx = zone_idx(zone); ++} ++ ++/* ++ * Builds allocation fallback zone lists. ++ * ++ * Add all populated zones of a node to the zonelist. ++ */ ++static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, ++ int nr_zones) ++{ ++ struct zone *zone; ++ enum zone_type zone_type = MAX_NR_ZONES; ++ ++ do { ++ zone_type--; ++ zone = pgdat->node_zones + zone_type; ++ if (populated_zone(zone)) { ++ zoneref_set_zone(zone, ++ &zonelist->_zonerefs[nr_zones++]); ++ check_highest_zone(zone_type); ++ } ++ } while (zone_type); ++ ++ return nr_zones; ++} ++ ++ ++/* ++ * zonelist_order: ++ * 0 = automatic detection of better ordering. ++ * 1 = order by ([node] distance, -zonetype) ++ * 2 = order by (-zonetype, [node] distance) ++ * ++ * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create ++ * the same zonelist. So only NUMA can configure this param. ++ */ ++#define ZONELIST_ORDER_DEFAULT 0 ++#define ZONELIST_ORDER_NODE 1 ++#define ZONELIST_ORDER_ZONE 2 ++ ++/* zonelist order in the kernel. ++ * set_zonelist_order() will set this to NODE or ZONE. ++ */ ++static int current_zonelist_order = ZONELIST_ORDER_DEFAULT; ++static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"}; ++ ++ ++#ifdef CONFIG_NUMA ++/* The value user specified ....changed by config */ ++static int user_zonelist_order = ZONELIST_ORDER_DEFAULT; ++/* string for sysctl */ ++#define NUMA_ZONELIST_ORDER_LEN 16 ++char numa_zonelist_order[16] = "default"; ++ ++/* ++ * interface for configure zonelist ordering. ++ * command line option "numa_zonelist_order" ++ * = "[dD]efault - default, automatic configuration. ++ * = "[nN]ode - order by node locality, then by zone within node ++ * = "[zZ]one - order by zone, then by locality within zone ++ */ ++ ++static int __parse_numa_zonelist_order(char *s) ++{ ++ if (*s == 'd' || *s == 'D') { ++ user_zonelist_order = ZONELIST_ORDER_DEFAULT; ++ } else if (*s == 'n' || *s == 'N') { ++ user_zonelist_order = ZONELIST_ORDER_NODE; ++ } else if (*s == 'z' || *s == 'Z') { ++ user_zonelist_order = ZONELIST_ORDER_ZONE; ++ } else { ++ printk(KERN_WARNING ++ "Ignoring invalid numa_zonelist_order value: " ++ "%s\n", s); ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++static __init int setup_numa_zonelist_order(char *s) ++{ ++ int ret; ++ ++ if (!s) ++ return 0; ++ ++ ret = __parse_numa_zonelist_order(s); ++ if (ret == 0) ++ strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN); ++ ++ return ret; ++} ++early_param("numa_zonelist_order", setup_numa_zonelist_order); ++ ++/* ++ * sysctl handler for numa_zonelist_order ++ */ ++int numa_zonelist_order_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, ++ loff_t *ppos) ++{ ++ char saved_string[NUMA_ZONELIST_ORDER_LEN]; ++ int ret; ++ static DEFINE_MUTEX(zl_order_mutex); ++ ++ mutex_lock(&zl_order_mutex); ++ if (write) { ++ if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) { ++ ret = -EINVAL; ++ goto out; ++ } ++ strcpy(saved_string, (char *)table->data); ++ } ++ ret = proc_dostring(table, write, buffer, length, ppos); ++ if (ret) ++ goto out; ++ if (write) { ++ int oldval = user_zonelist_order; ++ ++ ret = __parse_numa_zonelist_order((char *)table->data); ++ if (ret) { ++ /* ++ * bogus value. restore saved string ++ */ ++ strncpy((char *)table->data, saved_string, ++ NUMA_ZONELIST_ORDER_LEN); ++ user_zonelist_order = oldval; ++ } else if (oldval != user_zonelist_order) { ++ mutex_lock(&zonelists_mutex); ++ build_all_zonelists(NULL, NULL); ++ mutex_unlock(&zonelists_mutex); ++ } ++ } ++out: ++ mutex_unlock(&zl_order_mutex); ++ return ret; ++} ++ ++ ++#define MAX_NODE_LOAD (nr_online_nodes) ++static int node_load[MAX_NUMNODES]; ++ ++/** ++ * find_next_best_node - find the next node that should appear in a given node's fallback list ++ * @node: node whose fallback list we're appending ++ * @used_node_mask: nodemask_t of already used nodes ++ * ++ * We use a number of factors to determine which is the next node that should ++ * appear on a given node's fallback list. The node should not have appeared ++ * already in @node's fallback list, and it should be the next closest node ++ * according to the distance array (which contains arbitrary distance values ++ * from each node to each node in the system), and should also prefer nodes ++ * with no CPUs, since presumably they'll have very little allocation pressure ++ * on them otherwise. ++ * It returns -1 if no node is found. ++ */ ++static int find_next_best_node(int node, nodemask_t *used_node_mask) ++{ ++ int n, val; ++ int min_val = INT_MAX; ++ int best_node = NUMA_NO_NODE; ++ const struct cpumask *tmp = cpumask_of_node(0); ++ ++ /* Use the local node if we haven't already */ ++ if (!node_isset(node, *used_node_mask)) { ++ node_set(node, *used_node_mask); ++ return node; ++ } ++ ++ for_each_node_state(n, N_MEMORY) { ++ ++ /* Don't want a node to appear more than once */ ++ if (node_isset(n, *used_node_mask)) ++ continue; ++ ++ /* Use the distance array to find the distance */ ++ val = node_distance(node, n); ++ ++ /* Penalize nodes under us ("prefer the next node") */ ++ val += (n < node); ++ ++ /* Give preference to headless and unused nodes */ ++ tmp = cpumask_of_node(n); ++ if (!cpumask_empty(tmp)) ++ val += PENALTY_FOR_NODE_WITH_CPUS; ++ ++ /* Slight preference for less loaded node */ ++ val *= (MAX_NODE_LOAD*MAX_NUMNODES); ++ val += node_load[n]; ++ ++ if (val < min_val) { ++ min_val = val; ++ best_node = n; ++ } ++ } ++ ++ if (best_node >= 0) ++ node_set(best_node, *used_node_mask); ++ ++ return best_node; ++} ++ ++ ++/* ++ * Build zonelists ordered by node and zones within node. ++ * This results in maximum locality--normal zone overflows into local ++ * DMA zone, if any--but risks exhausting DMA zone. ++ */ ++static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) ++{ ++ int j; ++ struct zonelist *zonelist; ++ ++ zonelist = &pgdat->node_zonelists[0]; ++ for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++) ++ ; ++ j = build_zonelists_node(NODE_DATA(node), zonelist, j); ++ zonelist->_zonerefs[j].zone = NULL; ++ zonelist->_zonerefs[j].zone_idx = 0; ++} ++ ++/* ++ * Build gfp_thisnode zonelists ++ */ ++static void build_thisnode_zonelists(pg_data_t *pgdat) ++{ ++ int j; ++ struct zonelist *zonelist; ++ ++ zonelist = &pgdat->node_zonelists[1]; ++ j = build_zonelists_node(pgdat, zonelist, 0); ++ zonelist->_zonerefs[j].zone = NULL; ++ zonelist->_zonerefs[j].zone_idx = 0; ++} ++ ++/* ++ * Build zonelists ordered by zone and nodes within zones. ++ * This results in conserving DMA zone[s] until all Normal memory is ++ * exhausted, but results in overflowing to remote node while memory ++ * may still exist in local DMA zone. ++ */ ++static int node_order[MAX_NUMNODES]; ++ ++static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes) ++{ ++ int pos, j, node; ++ int zone_type; /* needs to be signed */ ++ struct zone *z; ++ struct zonelist *zonelist; ++ ++ zonelist = &pgdat->node_zonelists[0]; ++ pos = 0; ++ for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) { ++ for (j = 0; j < nr_nodes; j++) { ++ node = node_order[j]; ++ z = &NODE_DATA(node)->node_zones[zone_type]; ++ if (populated_zone(z)) { ++ zoneref_set_zone(z, ++ &zonelist->_zonerefs[pos++]); ++ check_highest_zone(zone_type); ++ } ++ } ++ } ++ zonelist->_zonerefs[pos].zone = NULL; ++ zonelist->_zonerefs[pos].zone_idx = 0; ++} ++ ++#if defined(CONFIG_64BIT) ++/* ++ * Devices that require DMA32/DMA are relatively rare and do not justify a ++ * penalty to every machine in case the specialised case applies. Default ++ * to Node-ordering on 64-bit NUMA machines ++ */ ++static int default_zonelist_order(void) ++{ ++ return ZONELIST_ORDER_NODE; ++} ++#else ++/* ++ * On 32-bit, the Normal zone needs to be preserved for allocations accessible ++ * by the kernel. If processes running on node 0 deplete the low memory zone ++ * then reclaim will occur more frequency increasing stalls and potentially ++ * be easier to OOM if a large percentage of the zone is under writeback or ++ * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set. ++ * Hence, default to zone ordering on 32-bit. ++ */ ++static int default_zonelist_order(void) ++{ ++ return ZONELIST_ORDER_ZONE; ++} ++#endif /* CONFIG_64BIT */ ++ ++static void set_zonelist_order(void) ++{ ++ if (user_zonelist_order == ZONELIST_ORDER_DEFAULT) ++ current_zonelist_order = default_zonelist_order(); ++ else ++ current_zonelist_order = user_zonelist_order; ++} ++ ++static void build_zonelists(pg_data_t *pgdat) ++{ ++ int j, node, load; ++ enum zone_type i; ++ nodemask_t used_mask; ++ int local_node, prev_node; ++ struct zonelist *zonelist; ++ int order = current_zonelist_order; ++ ++ /* initialize zonelists */ ++ for (i = 0; i < MAX_ZONELISTS; i++) { ++ zonelist = pgdat->node_zonelists + i; ++ zonelist->_zonerefs[0].zone = NULL; ++ zonelist->_zonerefs[0].zone_idx = 0; ++ } ++ ++ /* NUMA-aware ordering of nodes */ ++ local_node = pgdat->node_id; ++ load = nr_online_nodes; ++ prev_node = local_node; ++ nodes_clear(used_mask); ++ ++ memset(node_order, 0, sizeof(node_order)); ++ j = 0; ++ ++ while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { ++ /* ++ * We don't want to pressure a particular node. ++ * So adding penalty to the first node in same ++ * distance group to make it round-robin. ++ */ ++ if (node_distance(local_node, node) != ++ node_distance(local_node, prev_node)) ++ node_load[node] = load; ++ ++ prev_node = node; ++ load--; ++ if (order == ZONELIST_ORDER_NODE) ++ build_zonelists_in_node_order(pgdat, node); ++ else ++ node_order[j++] = node; /* remember order */ ++ } ++ ++ if (order == ZONELIST_ORDER_ZONE) { ++ /* calculate node order -- i.e., DMA last! */ ++ build_zonelists_in_zone_order(pgdat, j); ++ } ++ ++ build_thisnode_zonelists(pgdat); ++} ++ ++/* Construct the zonelist performance cache - see further mmzone.h */ ++static void build_zonelist_cache(pg_data_t *pgdat) ++{ ++ struct zonelist *zonelist; ++ struct zonelist_cache *zlc; ++ struct zoneref *z; ++ ++ zonelist = &pgdat->node_zonelists[0]; ++ zonelist->zlcache_ptr = zlc = &zonelist->zlcache; ++ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); ++ for (z = zonelist->_zonerefs; z->zone; z++) ++ zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z); ++} ++ ++#ifdef CONFIG_HAVE_MEMORYLESS_NODES ++/* ++ * Return node id of node used for "local" allocations. ++ * I.e., first node id of first zone in arg node's generic zonelist. ++ * Used for initializing percpu 'numa_mem', which is used primarily ++ * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. ++ */ ++int local_memory_node(int node) ++{ ++ struct zone *zone; ++ ++ (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL), ++ gfp_zone(GFP_KERNEL), ++ NULL, ++ &zone); ++ return zone->node; ++} ++#endif ++ ++#else /* CONFIG_NUMA */ ++ ++static void set_zonelist_order(void) ++{ ++ current_zonelist_order = ZONELIST_ORDER_ZONE; ++} ++ ++static void build_zonelists(pg_data_t *pgdat) ++{ ++ int node, local_node; ++ enum zone_type j; ++ struct zonelist *zonelist; ++ ++ local_node = pgdat->node_id; ++ ++ zonelist = &pgdat->node_zonelists[0]; ++ j = build_zonelists_node(pgdat, zonelist, 0); ++ ++ /* ++ * Now we build the zonelist so that it contains the zones ++ * of all the other nodes. ++ * We don't want to pressure a particular node, so when ++ * building the zones for node N, we make sure that the ++ * zones coming right after the local ones are those from ++ * node N+1 (modulo N) ++ */ ++ for (node = local_node + 1; node < MAX_NUMNODES; node++) { ++ if (!node_online(node)) ++ continue; ++ j = build_zonelists_node(NODE_DATA(node), zonelist, j); ++ } ++ for (node = 0; node < local_node; node++) { ++ if (!node_online(node)) ++ continue; ++ j = build_zonelists_node(NODE_DATA(node), zonelist, j); ++ } ++ ++ zonelist->_zonerefs[j].zone = NULL; ++ zonelist->_zonerefs[j].zone_idx = 0; ++} ++ ++/* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */ ++static void build_zonelist_cache(pg_data_t *pgdat) ++{ ++ pgdat->node_zonelists[0].zlcache_ptr = NULL; ++} ++ ++#endif /* CONFIG_NUMA */ ++ ++/* ++ * Boot pageset table. One per cpu which is going to be used for all ++ * zones and all nodes. The parameters will be set in such a way ++ * that an item put on a list will immediately be handed over to ++ * the buddy list. This is safe since pageset manipulation is done ++ * with interrupts disabled. ++ * ++ * The boot_pagesets must be kept even after bootup is complete for ++ * unused processors and/or zones. They do play a role for bootstrapping ++ * hotplugged processors. ++ * ++ * zoneinfo_show() and maybe other functions do ++ * not check if the processor is online before following the pageset pointer. ++ * Other parts of the kernel may not check if the zone is available. ++ */ ++static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); ++static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); ++static void setup_zone_pageset(struct zone *zone); ++ ++/* ++ * Global mutex to protect against size modification of zonelists ++ * as well as to serialize pageset setup for the new populated zone. ++ */ ++DEFINE_MUTEX(zonelists_mutex); ++ ++/* return values int ....just for stop_machine() */ ++static int __build_all_zonelists(void *data) ++{ ++ int nid; ++ int cpu; ++ pg_data_t *self = data; ++ ++#ifdef CONFIG_NUMA ++ memset(node_load, 0, sizeof(node_load)); ++#endif ++ ++ if (self && !node_online(self->node_id)) { ++ build_zonelists(self); ++ build_zonelist_cache(self); ++ } ++ ++ for_each_online_node(nid) { ++ pg_data_t *pgdat = NODE_DATA(nid); ++ ++ build_zonelists(pgdat); ++ build_zonelist_cache(pgdat); ++ } ++ ++ /* ++ * Initialize the boot_pagesets that are going to be used ++ * for bootstrapping processors. The real pagesets for ++ * each zone will be allocated later when the per cpu ++ * allocator is available. ++ * ++ * boot_pagesets are used also for bootstrapping offline ++ * cpus if the system is already booted because the pagesets ++ * are needed to initialize allocators on a specific cpu too. ++ * F.e. the percpu allocator needs the page allocator which ++ * needs the percpu allocator in order to allocate its pagesets ++ * (a chicken-egg dilemma). ++ */ ++ for_each_possible_cpu(cpu) { ++ setup_pageset(&per_cpu(boot_pageset, cpu), 0); ++ ++#ifdef CONFIG_HAVE_MEMORYLESS_NODES ++ /* ++ * We now know the "local memory node" for each node-- ++ * i.e., the node of the first zone in the generic zonelist. ++ * Set up numa_mem percpu variable for on-line cpus. During ++ * boot, only the boot cpu should be on-line; we'll init the ++ * secondary cpus' numa_mem as they come on-line. During ++ * node/memory hotplug, we'll fixup all on-line cpus. ++ */ ++ if (cpu_online(cpu)) ++ set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); ++#endif ++ } ++ ++ return 0; ++} ++ ++static noinline void __init ++build_all_zonelists_init(void) ++{ ++ __build_all_zonelists(NULL); ++ mminit_verify_zonelist(); ++ cpuset_init_current_mems_allowed(); ++} ++ ++/* ++ * Called with zonelists_mutex held always ++ * unless system_state == SYSTEM_BOOTING. ++ * ++ * __ref due to (1) call of __meminit annotated setup_zone_pageset ++ * [we're only called with non-NULL zone through __meminit paths] and ++ * (2) call of __init annotated helper build_all_zonelists_init ++ * [protected by SYSTEM_BOOTING]. ++ */ ++void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone) ++{ ++ set_zonelist_order(); ++ ++ if (system_state == SYSTEM_BOOTING) { ++ build_all_zonelists_init(); ++ } else { ++#ifdef CONFIG_MEMORY_HOTPLUG ++ if (zone) ++ setup_zone_pageset(zone); ++#endif ++ /* we have to stop all cpus to guarantee there is no user ++ of zonelist */ ++ stop_machine(__build_all_zonelists, pgdat, NULL); ++ /* cpuset refresh routine should be here */ ++ } ++ vm_total_pages = nr_free_pagecache_pages(); ++ /* ++ * Disable grouping by mobility if the number of pages in the ++ * system is too low to allow the mechanism to work. It would be ++ * more accurate, but expensive to check per-zone. This check is ++ * made on memory-hotadd so a system can start with mobility ++ * disabled and enable it later ++ */ ++ if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) ++ page_group_by_mobility_disabled = 1; ++ else ++ page_group_by_mobility_disabled = 0; ++ ++ pr_info("Built %i zonelists in %s order, mobility grouping %s. " ++ "Total pages: %ld\n", ++ nr_online_nodes, ++ zonelist_order_name[current_zonelist_order], ++ page_group_by_mobility_disabled ? "off" : "on", ++ vm_total_pages); ++#ifdef CONFIG_NUMA ++ pr_info("Policy zone: %s\n", zone_names[policy_zone]); ++#endif ++} ++ ++/* ++ * Helper functions to size the waitqueue hash table. ++ * Essentially these want to choose hash table sizes sufficiently ++ * large so that collisions trying to wait on pages are rare. ++ * But in fact, the number of active page waitqueues on typical ++ * systems is ridiculously low, less than 200. So this is even ++ * conservative, even though it seems large. ++ * ++ * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to ++ * waitqueues, i.e. the size of the waitq table given the number of pages. ++ */ ++#define PAGES_PER_WAITQUEUE 256 ++ ++#ifndef CONFIG_MEMORY_HOTPLUG ++static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) ++{ ++ unsigned long size = 1; ++ ++ pages /= PAGES_PER_WAITQUEUE; ++ ++ while (size < pages) ++ size <<= 1; ++ ++ /* ++ * Once we have dozens or even hundreds of threads sleeping ++ * on IO we've got bigger problems than wait queue collision. ++ * Limit the size of the wait table to a reasonable size. ++ */ ++ size = min(size, 4096UL); ++ ++ return max(size, 4UL); ++} ++#else ++/* ++ * A zone's size might be changed by hot-add, so it is not possible to determine ++ * a suitable size for its wait_table. So we use the maximum size now. ++ * ++ * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie: ++ * ++ * i386 (preemption config) : 4096 x 16 = 64Kbyte. ++ * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte. ++ * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte. ++ * ++ * The maximum entries are prepared when a zone's memory is (512K + 256) pages ++ * or more by the traditional way. (See above). It equals: ++ * ++ * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte. ++ * ia64(16K page size) : = ( 8G + 4M)byte. ++ * powerpc (64K page size) : = (32G +16M)byte. ++ */ ++static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) ++{ ++ return 4096UL; ++} ++#endif ++ ++/* ++ * This is an integer logarithm so that shifts can be used later ++ * to extract the more random high bits from the multiplicative ++ * hash function before the remainder is taken. ++ */ ++static inline unsigned long wait_table_bits(unsigned long size) ++{ ++ return ffz(~size); ++} ++ ++/* ++ * Check if a pageblock contains reserved pages ++ */ ++static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn) ++{ ++ unsigned long pfn; ++ ++ for (pfn = start_pfn; pfn < end_pfn; pfn++) { ++ if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn))) ++ return 1; ++ } ++ return 0; ++} ++ ++/* ++ * Mark a number of pageblocks as MIGRATE_RESERVE. The number ++ * of blocks reserved is based on min_wmark_pages(zone). The memory within ++ * the reserve will tend to store contiguous free pages. Setting min_free_kbytes ++ * higher will lead to a bigger reserve which will get freed as contiguous ++ * blocks as reclaim kicks in ++ */ ++static void setup_zone_migrate_reserve(struct zone *zone) ++{ ++ unsigned long start_pfn, pfn, end_pfn, block_end_pfn; ++ struct page *page; ++ unsigned long block_migratetype; ++ int reserve; ++ int old_reserve; ++ ++ /* ++ * Get the start pfn, end pfn and the number of blocks to reserve ++ * We have to be careful to be aligned to pageblock_nr_pages to ++ * make sure that we always check pfn_valid for the first page in ++ * the block. ++ */ ++ start_pfn = zone->zone_start_pfn; ++ end_pfn = zone_end_pfn(zone); ++ start_pfn = roundup(start_pfn, pageblock_nr_pages); ++ reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >> ++ pageblock_order; ++ ++ /* ++ * Reserve blocks are generally in place to help high-order atomic ++ * allocations that are short-lived. A min_free_kbytes value that ++ * would result in more than 2 reserve blocks for atomic allocations ++ * is assumed to be in place to help anti-fragmentation for the ++ * future allocation of hugepages at runtime. ++ */ ++ reserve = min(2, reserve); ++ old_reserve = zone->nr_migrate_reserve_block; ++ ++ /* When memory hot-add, we almost always need to do nothing */ ++ if (reserve == old_reserve) ++ return; ++ zone->nr_migrate_reserve_block = reserve; ++ ++ for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { ++ if (!pfn_valid(pfn)) ++ continue; ++ page = pfn_to_page(pfn); ++ ++ /* Watch out for overlapping nodes */ ++ if (page_to_nid(page) != zone_to_nid(zone)) ++ continue; ++ ++ block_migratetype = get_pageblock_migratetype(page); ++ ++ /* Only test what is necessary when the reserves are not met */ ++ if (reserve > 0) { ++ /* ++ * Blocks with reserved pages will never free, skip ++ * them. ++ */ ++ block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn); ++ if (pageblock_is_reserved(pfn, block_end_pfn)) ++ continue; ++ ++ /* If this block is reserved, account for it */ ++ if (block_migratetype == MIGRATE_RESERVE) { ++ reserve--; ++ continue; ++ } ++ ++ /* Suitable for reserving if this block is movable */ ++ if (block_migratetype == MIGRATE_MOVABLE) { ++ set_pageblock_migratetype(page, ++ MIGRATE_RESERVE); ++ move_freepages_block(zone, page, ++ MIGRATE_RESERVE); ++ reserve--; ++ continue; ++ } ++ } else if (!old_reserve) { ++ /* ++ * At boot time we don't need to scan the whole zone ++ * for turning off MIGRATE_RESERVE. ++ */ ++ break; ++ } ++ ++ /* ++ * If the reserve is met and this is a previous reserved block, ++ * take it back ++ */ ++ if (block_migratetype == MIGRATE_RESERVE) { ++ set_pageblock_migratetype(page, MIGRATE_MOVABLE); ++ move_freepages_block(zone, page, MIGRATE_MOVABLE); ++ } ++ } ++} ++ ++/* ++ * Initially all pages are reserved - free ones are freed ++ * up by free_all_bootmem() once the early boot process is ++ * done. Non-atomic initialization, single-pass. ++ */ ++void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, ++ unsigned long start_pfn, enum memmap_context context) ++{ ++ struct page *page; ++ unsigned long end_pfn = start_pfn + size; ++ unsigned long pfn; ++ struct zone *z; ++ ++ if (highest_memmap_pfn < end_pfn - 1) ++ highest_memmap_pfn = end_pfn - 1; ++ ++ z = &NODE_DATA(nid)->node_zones[zone]; ++ for (pfn = start_pfn; pfn < end_pfn; pfn++) { ++ /* ++ * There can be holes in boot-time mem_map[]s ++ * handed to this function. They do not ++ * exist on hotplugged memory. ++ */ ++ if (context == MEMMAP_EARLY) { ++ if (!early_pfn_valid(pfn)) ++ continue; ++ if (!early_pfn_in_nid(pfn, nid)) ++ continue; ++ } ++ page = pfn_to_page(pfn); ++ set_page_links(page, zone, nid, pfn); ++ mminit_verify_page_links(page, zone, nid, pfn); ++ init_page_count(page); ++ page_mapcount_reset(page); ++ page_cpupid_reset_last(page); ++ SetPageReserved(page); ++ /* ++ * Mark the block movable so that blocks are reserved for ++ * movable at startup. This will force kernel allocations ++ * to reserve their blocks rather than leaking throughout ++ * the address space during boot when many long-lived ++ * kernel allocations are made. Later some blocks near ++ * the start are marked MIGRATE_RESERVE by ++ * setup_zone_migrate_reserve() ++ * ++ * bitmap is created for zone's valid pfn range. but memmap ++ * can be created for invalid pages (for alignment) ++ * check here not to call set_pageblock_migratetype() against ++ * pfn out of zone. ++ */ ++ if ((z->zone_start_pfn <= pfn) ++ && (pfn < zone_end_pfn(z)) ++ && !(pfn & (pageblock_nr_pages - 1))) ++ set_pageblock_migratetype(page, MIGRATE_MOVABLE); ++ ++ INIT_LIST_HEAD(&page->lru); ++#ifdef WANT_PAGE_VIRTUAL ++ /* The shift won't overflow because ZONE_NORMAL is below 4G. */ ++ if (!is_highmem_idx(zone)) ++ set_page_address(page, __va(pfn << PAGE_SHIFT)); ++#endif ++ } ++} ++ ++static void __meminit zone_init_free_lists(struct zone *zone) ++{ ++ unsigned int order, t; ++ for_each_migratetype_order(order, t) { ++ INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); ++ zone->free_area[order].nr_free = 0; ++ } ++} ++ ++#ifndef __HAVE_ARCH_MEMMAP_INIT ++#define memmap_init(size, nid, zone, start_pfn) \ ++ memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY) ++#endif ++ ++static int zone_batchsize(struct zone *zone) ++{ ++#ifdef CONFIG_MMU ++ int batch; ++ ++ /* ++ * The per-cpu-pages pools are set to around 1000th of the ++ * size of the zone. But no more than 1/2 of a meg. ++ * ++ * OK, so we don't know how big the cache is. So guess. ++ */ ++ batch = zone->managed_pages / 1024; ++ if (batch * PAGE_SIZE > 512 * 1024) ++ batch = (512 * 1024) / PAGE_SIZE; ++ batch /= 4; /* We effectively *= 4 below */ ++ if (batch < 1) ++ batch = 1; ++ ++ /* ++ * Clamp the batch to a 2^n - 1 value. Having a power ++ * of 2 value was found to be more likely to have ++ * suboptimal cache aliasing properties in some cases. ++ * ++ * For example if 2 tasks are alternately allocating ++ * batches of pages, one task can end up with a lot ++ * of pages of one half of the possible page colors ++ * and the other with pages of the other colors. ++ */ ++ batch = rounddown_pow_of_two(batch + batch/2) - 1; ++ ++ return batch; ++ ++#else ++ /* The deferral and batching of frees should be suppressed under NOMMU ++ * conditions. ++ * ++ * The problem is that NOMMU needs to be able to allocate large chunks ++ * of contiguous memory as there's no hardware page translation to ++ * assemble apparent contiguous memory from discontiguous pages. ++ * ++ * Queueing large contiguous runs of pages for batching, however, ++ * causes the pages to actually be freed in smaller chunks. As there ++ * can be a significant delay between the individual batches being ++ * recycled, this leads to the once large chunks of space being ++ * fragmented and becoming unavailable for high-order allocations. ++ */ ++ return 0; ++#endif ++} ++ ++/* ++ * pcp->high and pcp->batch values are related and dependent on one another: ++ * ->batch must never be higher then ->high. ++ * The following function updates them in a safe manner without read side ++ * locking. ++ * ++ * Any new users of pcp->batch and pcp->high should ensure they can cope with ++ * those fields changing asynchronously (acording the the above rule). ++ * ++ * mutex_is_locked(&pcp_batch_high_lock) required when calling this function ++ * outside of boot time (or some other assurance that no concurrent updaters ++ * exist). ++ */ ++static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, ++ unsigned long batch) ++{ ++ /* start with a fail safe value for batch */ ++ pcp->batch = 1; ++ smp_wmb(); ++ ++ /* Update high, then batch, in order */ ++ pcp->high = high; ++ smp_wmb(); ++ ++ pcp->batch = batch; ++} ++ ++/* a companion to pageset_set_high() */ ++static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) ++{ ++ pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); ++} ++ ++static void pageset_init(struct per_cpu_pageset *p) ++{ ++ struct per_cpu_pages *pcp; ++ int migratetype; ++ ++ memset(p, 0, sizeof(*p)); ++ ++ pcp = &p->pcp; ++ pcp->count = 0; ++ for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) ++ INIT_LIST_HEAD(&pcp->lists[migratetype]); ++} ++ ++static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) ++{ ++ pageset_init(p); ++ pageset_set_batch(p, batch); ++} ++ ++/* ++ * pageset_set_high() sets the high water mark for hot per_cpu_pagelist ++ * to the value high for the pageset p. ++ */ ++static void pageset_set_high(struct per_cpu_pageset *p, ++ unsigned long high) ++{ ++ unsigned long batch = max(1UL, high / 4); ++ if ((high / 4) > (PAGE_SHIFT * 8)) ++ batch = PAGE_SHIFT * 8; ++ ++ pageset_update(&p->pcp, high, batch); ++} ++ ++static void pageset_set_high_and_batch(struct zone *zone, ++ struct per_cpu_pageset *pcp) ++{ ++ if (percpu_pagelist_fraction) ++ pageset_set_high(pcp, ++ (zone->managed_pages / ++ percpu_pagelist_fraction)); ++ else ++ pageset_set_batch(pcp, zone_batchsize(zone)); ++} ++ ++static void __meminit zone_pageset_init(struct zone *zone, int cpu) ++{ ++ struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); ++ ++ pageset_init(pcp); ++ pageset_set_high_and_batch(zone, pcp); ++} ++ ++static void __meminit setup_zone_pageset(struct zone *zone) ++{ ++ int cpu; ++ zone->pageset = alloc_percpu(struct per_cpu_pageset); ++ for_each_possible_cpu(cpu) ++ zone_pageset_init(zone, cpu); ++} ++ ++/* ++ * Allocate per cpu pagesets and initialize them. ++ * Before this call only boot pagesets were available. ++ */ ++void __init setup_per_cpu_pageset(void) ++{ ++ struct zone *zone; ++ ++ for_each_populated_zone(zone) ++ setup_zone_pageset(zone); ++} ++ ++static noinline __init_refok ++int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) ++{ ++ int i; ++ size_t alloc_size; ++ ++ /* ++ * The per-page waitqueue mechanism uses hashed waitqueues ++ * per zone. ++ */ ++ zone->wait_table_hash_nr_entries = ++ wait_table_hash_nr_entries(zone_size_pages); ++ zone->wait_table_bits = ++ wait_table_bits(zone->wait_table_hash_nr_entries); ++ alloc_size = zone->wait_table_hash_nr_entries ++ * sizeof(wait_queue_head_t); ++ ++ if (!slab_is_available()) { ++ zone->wait_table = (wait_queue_head_t *) ++ memblock_virt_alloc_node_nopanic( ++ alloc_size, zone->zone_pgdat->node_id); ++ } else { ++ /* ++ * This case means that a zone whose size was 0 gets new memory ++ * via memory hot-add. ++ * But it may be the case that a new node was hot-added. In ++ * this case vmalloc() will not be able to use this new node's ++ * memory - this wait_table must be initialized to use this new ++ * node itself as well. ++ * To use this new node's memory, further consideration will be ++ * necessary. ++ */ ++ zone->wait_table = vmalloc(alloc_size); ++ } ++ if (!zone->wait_table) ++ return -ENOMEM; ++ ++ for (i = 0; i < zone->wait_table_hash_nr_entries; ++i) ++ init_waitqueue_head(zone->wait_table + i); ++ ++ return 0; ++} ++ ++static __meminit void zone_pcp_init(struct zone *zone) ++{ ++ /* ++ * per cpu subsystem is not up at this point. The following code ++ * relies on the ability of the linker to provide the ++ * offset of a (static) per cpu variable into the per cpu area. ++ */ ++ zone->pageset = &boot_pageset; ++ ++ if (populated_zone(zone)) ++ printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", ++ zone->name, zone->present_pages, ++ zone_batchsize(zone)); ++} ++ ++int __meminit init_currently_empty_zone(struct zone *zone, ++ unsigned long zone_start_pfn, ++ unsigned long size, ++ enum memmap_context context) ++{ ++ struct pglist_data *pgdat = zone->zone_pgdat; ++ int ret; ++ ret = zone_wait_table_init(zone, size); ++ if (ret) ++ return ret; ++ pgdat->nr_zones = zone_idx(zone) + 1; ++ ++ zone->zone_start_pfn = zone_start_pfn; ++ ++ mminit_dprintk(MMINIT_TRACE, "memmap_init", ++ "Initialising map node %d zone %lu pfns %lu -> %lu\n", ++ pgdat->node_id, ++ (unsigned long)zone_idx(zone), ++ zone_start_pfn, (zone_start_pfn + size)); ++ ++ zone_init_free_lists(zone); ++ ++ return 0; ++} ++ ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID ++/* ++ * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. ++ */ ++int __meminit __early_pfn_to_nid(unsigned long pfn) ++{ ++ unsigned long start_pfn, end_pfn; ++ int nid; ++ /* ++ * NOTE: The following SMP-unsafe globals are only used early in boot ++ * when the kernel is running single-threaded. ++ */ ++ static unsigned long __meminitdata last_start_pfn, last_end_pfn; ++ static int __meminitdata last_nid; ++ ++ if (last_start_pfn <= pfn && pfn < last_end_pfn) ++ return last_nid; ++ ++ nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); ++ if (nid != -1) { ++ last_start_pfn = start_pfn; ++ last_end_pfn = end_pfn; ++ last_nid = nid; ++ } ++ ++ return nid; ++} ++#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ ++ ++int __meminit early_pfn_to_nid(unsigned long pfn) ++{ ++ int nid; ++ ++ nid = __early_pfn_to_nid(pfn); ++ if (nid >= 0) ++ return nid; ++ /* just returns 0 */ ++ return 0; ++} ++ ++#ifdef CONFIG_NODES_SPAN_OTHER_NODES ++bool __meminit early_pfn_in_nid(unsigned long pfn, int node) ++{ ++ int nid; ++ ++ nid = __early_pfn_to_nid(pfn); ++ if (nid >= 0 && nid != node) ++ return false; ++ return true; ++} ++#endif ++ ++/** ++ * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range ++ * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. ++ * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid ++ * ++ * If an architecture guarantees that all ranges registered contain no holes ++ * and may be freed, this this function may be used instead of calling ++ * memblock_free_early_nid() manually. ++ */ ++void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) ++{ ++ unsigned long start_pfn, end_pfn; ++ int i, this_nid; ++ ++ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { ++ start_pfn = min(start_pfn, max_low_pfn); ++ end_pfn = min(end_pfn, max_low_pfn); ++ ++ if (start_pfn < end_pfn) ++ memblock_free_early_nid(PFN_PHYS(start_pfn), ++ (end_pfn - start_pfn) << PAGE_SHIFT, ++ this_nid); ++ } ++} ++ ++/** ++ * sparse_memory_present_with_active_regions - Call memory_present for each active range ++ * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. ++ * ++ * If an architecture guarantees that all ranges registered contain no holes and may ++ * be freed, this function may be used instead of calling memory_present() manually. ++ */ ++void __init sparse_memory_present_with_active_regions(int nid) ++{ ++ unsigned long start_pfn, end_pfn; ++ int i, this_nid; ++ ++ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) ++ memory_present(this_nid, start_pfn, end_pfn); ++} ++ ++/** ++ * get_pfn_range_for_nid - Return the start and end page frames for a node ++ * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. ++ * @start_pfn: Passed by reference. On return, it will have the node start_pfn. ++ * @end_pfn: Passed by reference. On return, it will have the node end_pfn. ++ * ++ * It returns the start and end page frame of a node based on information ++ * provided by memblock_set_node(). If called for a node ++ * with no available memory, a warning is printed and the start and end ++ * PFNs will be 0. ++ */ ++void __meminit get_pfn_range_for_nid(unsigned int nid, ++ unsigned long *start_pfn, unsigned long *end_pfn) ++{ ++ unsigned long this_start_pfn, this_end_pfn; ++ int i; ++ ++ *start_pfn = -1UL; ++ *end_pfn = 0; ++ ++ for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { ++ *start_pfn = min(*start_pfn, this_start_pfn); ++ *end_pfn = max(*end_pfn, this_end_pfn); ++ } ++ ++ if (*start_pfn == -1UL) ++ *start_pfn = 0; ++} ++ ++/* ++ * This finds a zone that can be used for ZONE_MOVABLE pages. The ++ * assumption is made that zones within a node are ordered in monotonic ++ * increasing memory addresses so that the "highest" populated zone is used ++ */ ++static void __init find_usable_zone_for_movable(void) ++{ ++ int zone_index; ++ for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { ++ if (zone_index == ZONE_MOVABLE) ++ continue; ++ ++ if (arch_zone_highest_possible_pfn[zone_index] > ++ arch_zone_lowest_possible_pfn[zone_index]) ++ break; ++ } ++ ++ VM_BUG_ON(zone_index == -1); ++ movable_zone = zone_index; ++} ++ ++/* ++ * The zone ranges provided by the architecture do not include ZONE_MOVABLE ++ * because it is sized independent of architecture. Unlike the other zones, ++ * the starting point for ZONE_MOVABLE is not fixed. It may be different ++ * in each node depending on the size of each node and how evenly kernelcore ++ * is distributed. This helper function adjusts the zone ranges ++ * provided by the architecture for a given node by using the end of the ++ * highest usable zone for ZONE_MOVABLE. This preserves the assumption that ++ * zones within a node are in order of monotonic increases memory addresses ++ */ ++static void __meminit adjust_zone_range_for_zone_movable(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *zone_start_pfn, ++ unsigned long *zone_end_pfn) ++{ ++ /* Only adjust if ZONE_MOVABLE is on this node */ ++ if (zone_movable_pfn[nid]) { ++ /* Size ZONE_MOVABLE */ ++ if (zone_type == ZONE_MOVABLE) { ++ *zone_start_pfn = zone_movable_pfn[nid]; ++ *zone_end_pfn = min(node_end_pfn, ++ arch_zone_highest_possible_pfn[movable_zone]); ++ ++ /* Adjust for ZONE_MOVABLE starting within this range */ ++ } else if (*zone_start_pfn < zone_movable_pfn[nid] && ++ *zone_end_pfn > zone_movable_pfn[nid]) { ++ *zone_end_pfn = zone_movable_pfn[nid]; ++ ++ /* Check if this whole range is within ZONE_MOVABLE */ ++ } else if (*zone_start_pfn >= zone_movable_pfn[nid]) ++ *zone_start_pfn = *zone_end_pfn; ++ } ++} ++ ++/* ++ * Return the number of pages a zone spans in a node, including holes ++ * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() ++ */ ++static unsigned long __meminit zone_spanned_pages_in_node(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *ignored) ++{ ++ unsigned long zone_start_pfn, zone_end_pfn; ++ ++ /* Get the start and end of the zone */ ++ zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; ++ zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; ++ adjust_zone_range_for_zone_movable(nid, zone_type, ++ node_start_pfn, node_end_pfn, ++ &zone_start_pfn, &zone_end_pfn); ++ ++ /* Check that this node has pages within the zone's required range */ ++ if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn) ++ return 0; ++ ++ /* Move the zone boundaries inside the node if necessary */ ++ zone_end_pfn = min(zone_end_pfn, node_end_pfn); ++ zone_start_pfn = max(zone_start_pfn, node_start_pfn); ++ ++ /* Return the spanned pages */ ++ return zone_end_pfn - zone_start_pfn; ++} ++ ++/* ++ * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, ++ * then all holes in the requested range will be accounted for. ++ */ ++unsigned long __meminit __absent_pages_in_range(int nid, ++ unsigned long range_start_pfn, ++ unsigned long range_end_pfn) ++{ ++ unsigned long nr_absent = range_end_pfn - range_start_pfn; ++ unsigned long start_pfn, end_pfn; ++ int i; ++ ++ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { ++ start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); ++ end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); ++ nr_absent -= end_pfn - start_pfn; ++ } ++ return nr_absent; ++} ++ ++/** ++ * absent_pages_in_range - Return number of page frames in holes within a range ++ * @start_pfn: The start PFN to start searching for holes ++ * @end_pfn: The end PFN to stop searching for holes ++ * ++ * It returns the number of pages frames in memory holes within a range. ++ */ ++unsigned long __init absent_pages_in_range(unsigned long start_pfn, ++ unsigned long end_pfn) ++{ ++ return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); ++} ++ ++/* Return the number of page frames in holes in a zone on a node */ ++static unsigned long __meminit zone_absent_pages_in_node(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *ignored) ++{ ++ unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; ++ unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; ++ unsigned long zone_start_pfn, zone_end_pfn; ++ ++ zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); ++ zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); ++ ++ adjust_zone_range_for_zone_movable(nid, zone_type, ++ node_start_pfn, node_end_pfn, ++ &zone_start_pfn, &zone_end_pfn); ++ return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); ++} ++ ++#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *zones_size) ++{ ++ return zones_size[zone_type]; ++} ++ ++static inline unsigned long __meminit zone_absent_pages_in_node(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *zholes_size) ++{ ++ if (!zholes_size) ++ return 0; ++ ++ return zholes_size[zone_type]; ++} ++ ++#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++ ++static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *zones_size, ++ unsigned long *zholes_size) ++{ ++ unsigned long realtotalpages, totalpages = 0; ++ enum zone_type i; ++ ++ for (i = 0; i < MAX_NR_ZONES; i++) ++ totalpages += zone_spanned_pages_in_node(pgdat->node_id, i, ++ node_start_pfn, ++ node_end_pfn, ++ zones_size); ++ pgdat->node_spanned_pages = totalpages; ++ ++ realtotalpages = totalpages; ++ for (i = 0; i < MAX_NR_ZONES; i++) ++ realtotalpages -= ++ zone_absent_pages_in_node(pgdat->node_id, i, ++ node_start_pfn, node_end_pfn, ++ zholes_size); ++ pgdat->node_present_pages = realtotalpages; ++ printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, ++ realtotalpages); ++} ++ ++#ifndef CONFIG_SPARSEMEM ++/* ++ * Calculate the size of the zone->blockflags rounded to an unsigned long ++ * Start by making sure zonesize is a multiple of pageblock_order by rounding ++ * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally ++ * round what is now in bits to nearest long in bits, then return it in ++ * bytes. ++ */ ++static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) ++{ ++ unsigned long usemapsize; ++ ++ zonesize += zone_start_pfn & (pageblock_nr_pages-1); ++ usemapsize = roundup(zonesize, pageblock_nr_pages); ++ usemapsize = usemapsize >> pageblock_order; ++ usemapsize *= NR_PAGEBLOCK_BITS; ++ usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); ++ ++ return usemapsize / 8; ++} ++ ++static void __init setup_usemap(struct pglist_data *pgdat, ++ struct zone *zone, ++ unsigned long zone_start_pfn, ++ unsigned long zonesize) ++{ ++ unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); ++ zone->pageblock_flags = NULL; ++ if (usemapsize) ++ zone->pageblock_flags = ++ memblock_virt_alloc_node_nopanic(usemapsize, ++ pgdat->node_id); ++} ++#else ++static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, ++ unsigned long zone_start_pfn, unsigned long zonesize) {} ++#endif /* CONFIG_SPARSEMEM */ ++ ++#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE ++ ++/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ ++void __paginginit set_pageblock_order(void) ++{ ++ unsigned int order; ++ ++ /* Check that pageblock_nr_pages has not already been setup */ ++ if (pageblock_order) ++ return; ++ ++ if (HPAGE_SHIFT > PAGE_SHIFT) ++ order = HUGETLB_PAGE_ORDER; ++ else ++ order = MAX_ORDER - 1; ++ ++ /* ++ * Assume the largest contiguous order of interest is a huge page. ++ * This value may be variable depending on boot parameters on IA64 and ++ * powerpc. ++ */ ++ pageblock_order = order; ++} ++#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ ++ ++/* ++ * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() ++ * is unused as pageblock_order is set at compile-time. See ++ * include/linux/pageblock-flags.h for the values of pageblock_order based on ++ * the kernel config ++ */ ++void __paginginit set_pageblock_order(void) ++{ ++} ++ ++#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ ++ ++static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages, ++ unsigned long present_pages) ++{ ++ unsigned long pages = spanned_pages; ++ ++ /* ++ * Provide a more accurate estimation if there are holes within ++ * the zone and SPARSEMEM is in use. If there are holes within the ++ * zone, each populated memory region may cost us one or two extra ++ * memmap pages due to alignment because memmap pages for each ++ * populated regions may not naturally algined on page boundary. ++ * So the (present_pages >> 4) heuristic is a tradeoff for that. ++ */ ++ if (spanned_pages > present_pages + (present_pages >> 4) && ++ IS_ENABLED(CONFIG_SPARSEMEM)) ++ pages = present_pages; ++ ++ return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; ++} ++ ++/* ++ * Set up the zone data structures: ++ * - mark all pages reserved ++ * - mark all memory queues empty ++ * - clear the memory bitmaps ++ * ++ * NOTE: pgdat should get zeroed by caller. ++ */ ++static void __paginginit free_area_init_core(struct pglist_data *pgdat, ++ unsigned long node_start_pfn, unsigned long node_end_pfn, ++ unsigned long *zones_size, unsigned long *zholes_size) ++{ ++ enum zone_type j; ++ int nid = pgdat->node_id; ++ unsigned long zone_start_pfn = pgdat->node_start_pfn; ++ int ret; ++ ++ pgdat_resize_init(pgdat); ++#ifdef CONFIG_NUMA_BALANCING ++ spin_lock_init(&pgdat->numabalancing_migrate_lock); ++ pgdat->numabalancing_migrate_nr_pages = 0; ++ pgdat->numabalancing_migrate_next_window = jiffies; ++#endif ++ init_waitqueue_head(&pgdat->kswapd_wait); ++ init_waitqueue_head(&pgdat->pfmemalloc_wait); ++ pgdat_page_ext_init(pgdat); ++ ++ for (j = 0; j < MAX_NR_ZONES; j++) { ++ struct zone *zone = pgdat->node_zones + j; ++ unsigned long size, realsize, freesize, memmap_pages; ++ ++ size = zone_spanned_pages_in_node(nid, j, node_start_pfn, ++ node_end_pfn, zones_size); ++ realsize = freesize = size - zone_absent_pages_in_node(nid, j, ++ node_start_pfn, ++ node_end_pfn, ++ zholes_size); ++ ++ /* ++ * Adjust freesize so that it accounts for how much memory ++ * is used by this zone for memmap. This affects the watermark ++ * and per-cpu initialisations ++ */ ++ memmap_pages = calc_memmap_size(size, realsize); ++ if (!is_highmem_idx(j)) { ++ if (freesize >= memmap_pages) { ++ freesize -= memmap_pages; ++ if (memmap_pages) ++ printk(KERN_DEBUG ++ " %s zone: %lu pages used for memmap\n", ++ zone_names[j], memmap_pages); ++ } else ++ printk(KERN_WARNING ++ " %s zone: %lu pages exceeds freesize %lu\n", ++ zone_names[j], memmap_pages, freesize); ++ } ++ ++ /* Account for reserved pages */ ++ if (j == 0 && freesize > dma_reserve) { ++ freesize -= dma_reserve; ++ printk(KERN_DEBUG " %s zone: %lu pages reserved\n", ++ zone_names[0], dma_reserve); ++ } ++ ++ if (!is_highmem_idx(j)) ++ nr_kernel_pages += freesize; ++ /* Charge for highmem memmap if there are enough kernel pages */ ++ else if (nr_kernel_pages > memmap_pages * 2) ++ nr_kernel_pages -= memmap_pages; ++ nr_all_pages += freesize; ++ ++ zone->spanned_pages = size; ++ zone->present_pages = realsize; ++ /* ++ * Set an approximate value for lowmem here, it will be adjusted ++ * when the bootmem allocator frees pages into the buddy system. ++ * And all highmem pages will be managed by the buddy system. ++ */ ++ zone->managed_pages = is_highmem_idx(j) ? realsize : freesize; ++#ifdef CONFIG_NUMA ++ zone->node = nid; ++ zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio) ++ / 100; ++ zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100; ++#endif ++ zone->name = zone_names[j]; ++ spin_lock_init(&zone->lock); ++ spin_lock_init(&zone->lru_lock); ++ zone_seqlock_init(zone); ++ zone->zone_pgdat = pgdat; ++ zone_pcp_init(zone); ++ ++ /* For bootup, initialized properly in watermark setup */ ++ mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages); ++ ++ lruvec_init(&zone->lruvec); ++ if (!size) ++ continue; ++ ++ set_pageblock_order(); ++ setup_usemap(pgdat, zone, zone_start_pfn, size); ++ ret = init_currently_empty_zone(zone, zone_start_pfn, ++ size, MEMMAP_EARLY); ++ BUG_ON(ret); ++ memmap_init(size, nid, j, zone_start_pfn); ++ zone_start_pfn += size; ++ } ++} ++ ++static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat) ++{ ++ /* Skip empty nodes */ ++ if (!pgdat->node_spanned_pages) ++ return; ++ ++#ifdef CONFIG_FLAT_NODE_MEM_MAP ++ /* ia64 gets its own node_mem_map, before this, without bootmem */ ++ if (!pgdat->node_mem_map) { ++ unsigned long size, start, end; ++ struct page *map; ++ ++ /* ++ * The zone's endpoints aren't required to be MAX_ORDER ++ * aligned but the node_mem_map endpoints must be in order ++ * for the buddy allocator to function correctly. ++ */ ++ start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); ++ end = pgdat_end_pfn(pgdat); ++ end = ALIGN(end, MAX_ORDER_NR_PAGES); ++ size = (end - start) * sizeof(struct page); ++ map = alloc_remap(pgdat->node_id, size); ++ if (!map) ++ map = memblock_virt_alloc_node_nopanic(size, ++ pgdat->node_id); ++ pgdat->node_mem_map = map + (pgdat->node_start_pfn - start); ++ } ++#ifndef CONFIG_NEED_MULTIPLE_NODES ++ /* ++ * With no DISCONTIG, the global mem_map is just set as node 0's ++ */ ++ if (pgdat == NODE_DATA(0)) { ++ mem_map = NODE_DATA(0)->node_mem_map; ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++ if (page_to_pfn(mem_map) != pgdat->node_start_pfn) ++ mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET); ++#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++ } ++#endif ++#endif /* CONFIG_FLAT_NODE_MEM_MAP */ ++} ++ ++void __paginginit free_area_init_node(int nid, unsigned long *zones_size, ++ unsigned long node_start_pfn, unsigned long *zholes_size) ++{ ++ pg_data_t *pgdat = NODE_DATA(nid); ++ unsigned long start_pfn = 0; ++ unsigned long end_pfn = 0; ++ ++ /* pg_data_t should be reset to zero when it's allocated */ ++ WARN_ON(pgdat->nr_zones || pgdat->classzone_idx); ++ ++ pgdat->node_id = nid; ++ pgdat->node_start_pfn = node_start_pfn; ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++ get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); ++ pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, ++ (u64)start_pfn << PAGE_SHIFT, ((u64)end_pfn << PAGE_SHIFT) - 1); ++#endif ++ calculate_node_totalpages(pgdat, start_pfn, end_pfn, ++ zones_size, zholes_size); ++ ++ alloc_node_mem_map(pgdat); ++#ifdef CONFIG_FLAT_NODE_MEM_MAP ++ printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n", ++ nid, (unsigned long)pgdat, ++ (unsigned long)pgdat->node_mem_map); ++#endif ++ ++ free_area_init_core(pgdat, start_pfn, end_pfn, ++ zones_size, zholes_size); ++} ++ ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++ ++#if MAX_NUMNODES > 1 ++/* ++ * Figure out the number of possible node ids. ++ */ ++void __init setup_nr_node_ids(void) ++{ ++ unsigned int node; ++ unsigned int highest = 0; ++ ++ for_each_node_mask(node, node_possible_map) ++ highest = node; ++ nr_node_ids = highest + 1; ++} ++#endif ++ ++/** ++ * node_map_pfn_alignment - determine the maximum internode alignment ++ * ++ * This function should be called after node map is populated and sorted. ++ * It calculates the maximum power of two alignment which can distinguish ++ * all the nodes. ++ * ++ * For example, if all nodes are 1GiB and aligned to 1GiB, the return value ++ * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the ++ * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is ++ * shifted, 1GiB is enough and this function will indicate so. ++ * ++ * This is used to test whether pfn -> nid mapping of the chosen memory ++ * model has fine enough granularity to avoid incorrect mapping for the ++ * populated node map. ++ * ++ * Returns the determined alignment in pfn's. 0 if there is no alignment ++ * requirement (single node). ++ */ ++unsigned long __init node_map_pfn_alignment(void) ++{ ++ unsigned long accl_mask = 0, last_end = 0; ++ unsigned long start, end, mask; ++ int last_nid = -1; ++ int i, nid; ++ ++ for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { ++ if (!start || last_nid < 0 || last_nid == nid) { ++ last_nid = nid; ++ last_end = end; ++ continue; ++ } ++ ++ /* ++ * Start with a mask granular enough to pin-point to the ++ * start pfn and tick off bits one-by-one until it becomes ++ * too coarse to separate the current node from the last. ++ */ ++ mask = ~((1 << __ffs(start)) - 1); ++ while (mask && last_end <= (start & (mask << 1))) ++ mask <<= 1; ++ ++ /* accumulate all internode masks */ ++ accl_mask |= mask; ++ } ++ ++ /* convert mask to number of pages */ ++ return ~accl_mask + 1; ++} ++ ++/* Find the lowest pfn for a node */ ++static unsigned long __init find_min_pfn_for_node(int nid) ++{ ++ unsigned long min_pfn = ULONG_MAX; ++ unsigned long start_pfn; ++ int i; ++ ++ for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) ++ min_pfn = min(min_pfn, start_pfn); ++ ++ if (min_pfn == ULONG_MAX) { ++ printk(KERN_WARNING ++ "Could not find start_pfn for node %d\n", nid); ++ return 0; ++ } ++ ++ return min_pfn; ++} ++ ++/** ++ * find_min_pfn_with_active_regions - Find the minimum PFN registered ++ * ++ * It returns the minimum PFN based on information provided via ++ * memblock_set_node(). ++ */ ++unsigned long __init find_min_pfn_with_active_regions(void) ++{ ++ return find_min_pfn_for_node(MAX_NUMNODES); ++} ++ ++/* ++ * early_calculate_totalpages() ++ * Sum pages in active regions for movable zone. ++ * Populate N_MEMORY for calculating usable_nodes. ++ */ ++static unsigned long __init early_calculate_totalpages(void) ++{ ++ unsigned long totalpages = 0; ++ unsigned long start_pfn, end_pfn; ++ int i, nid; ++ ++ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { ++ unsigned long pages = end_pfn - start_pfn; ++ ++ totalpages += pages; ++ if (pages) ++ node_set_state(nid, N_MEMORY); ++ } ++ return totalpages; ++} ++ ++/* ++ * Find the PFN the Movable zone begins in each node. Kernel memory ++ * is spread evenly between nodes as long as the nodes have enough ++ * memory. When they don't, some nodes will have more kernelcore than ++ * others ++ */ ++static void __init find_zone_movable_pfns_for_nodes(void) ++{ ++ int i, nid; ++ unsigned long usable_startpfn; ++ unsigned long kernelcore_node, kernelcore_remaining; ++ /* save the state before borrow the nodemask */ ++ nodemask_t saved_node_state = node_states[N_MEMORY]; ++ unsigned long totalpages = early_calculate_totalpages(); ++ int usable_nodes = nodes_weight(node_states[N_MEMORY]); ++ struct memblock_region *r; ++ ++ /* Need to find movable_zone earlier when movable_node is specified. */ ++ find_usable_zone_for_movable(); ++ ++ /* ++ * If movable_node is specified, ignore kernelcore and movablecore ++ * options. ++ */ ++ if (movable_node_is_enabled()) { ++ for_each_memblock(memory, r) { ++ if (!memblock_is_hotpluggable(r)) ++ continue; ++ ++ nid = r->nid; ++ ++ usable_startpfn = PFN_DOWN(r->base); ++ zone_movable_pfn[nid] = zone_movable_pfn[nid] ? ++ min(usable_startpfn, zone_movable_pfn[nid]) : ++ usable_startpfn; ++ } ++ ++ goto out2; ++ } ++ ++ /* ++ * If movablecore=nn[KMG] was specified, calculate what size of ++ * kernelcore that corresponds so that memory usable for ++ * any allocation type is evenly spread. If both kernelcore ++ * and movablecore are specified, then the value of kernelcore ++ * will be used for required_kernelcore if it's greater than ++ * what movablecore would have allowed. ++ */ ++ if (required_movablecore) { ++ unsigned long corepages; ++ ++ /* ++ * Round-up so that ZONE_MOVABLE is at least as large as what ++ * was requested by the user ++ */ ++ required_movablecore = ++ roundup(required_movablecore, MAX_ORDER_NR_PAGES); ++ corepages = totalpages - required_movablecore; ++ ++ required_kernelcore = max(required_kernelcore, corepages); ++ } ++ ++ /* If kernelcore was not specified, there is no ZONE_MOVABLE */ ++ if (!required_kernelcore) ++ goto out; ++ ++ /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ ++ usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; ++ ++restart: ++ /* Spread kernelcore memory as evenly as possible throughout nodes */ ++ kernelcore_node = required_kernelcore / usable_nodes; ++ for_each_node_state(nid, N_MEMORY) { ++ unsigned long start_pfn, end_pfn; ++ ++ /* ++ * Recalculate kernelcore_node if the division per node ++ * now exceeds what is necessary to satisfy the requested ++ * amount of memory for the kernel ++ */ ++ if (required_kernelcore < kernelcore_node) ++ kernelcore_node = required_kernelcore / usable_nodes; ++ ++ /* ++ * As the map is walked, we track how much memory is usable ++ * by the kernel using kernelcore_remaining. When it is ++ * 0, the rest of the node is usable by ZONE_MOVABLE ++ */ ++ kernelcore_remaining = kernelcore_node; ++ ++ /* Go through each range of PFNs within this node */ ++ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { ++ unsigned long size_pages; ++ ++ start_pfn = max(start_pfn, zone_movable_pfn[nid]); ++ if (start_pfn >= end_pfn) ++ continue; ++ ++ /* Account for what is only usable for kernelcore */ ++ if (start_pfn < usable_startpfn) { ++ unsigned long kernel_pages; ++ kernel_pages = min(end_pfn, usable_startpfn) ++ - start_pfn; ++ ++ kernelcore_remaining -= min(kernel_pages, ++ kernelcore_remaining); ++ required_kernelcore -= min(kernel_pages, ++ required_kernelcore); ++ ++ /* Continue if range is now fully accounted */ ++ if (end_pfn <= usable_startpfn) { ++ ++ /* ++ * Push zone_movable_pfn to the end so ++ * that if we have to rebalance ++ * kernelcore across nodes, we will ++ * not double account here ++ */ ++ zone_movable_pfn[nid] = end_pfn; ++ continue; ++ } ++ start_pfn = usable_startpfn; ++ } ++ ++ /* ++ * The usable PFN range for ZONE_MOVABLE is from ++ * start_pfn->end_pfn. Calculate size_pages as the ++ * number of pages used as kernelcore ++ */ ++ size_pages = end_pfn - start_pfn; ++ if (size_pages > kernelcore_remaining) ++ size_pages = kernelcore_remaining; ++ zone_movable_pfn[nid] = start_pfn + size_pages; ++ ++ /* ++ * Some kernelcore has been met, update counts and ++ * break if the kernelcore for this node has been ++ * satisfied ++ */ ++ required_kernelcore -= min(required_kernelcore, ++ size_pages); ++ kernelcore_remaining -= size_pages; ++ if (!kernelcore_remaining) ++ break; ++ } ++ } ++ ++ /* ++ * If there is still required_kernelcore, we do another pass with one ++ * less node in the count. This will push zone_movable_pfn[nid] further ++ * along on the nodes that still have memory until kernelcore is ++ * satisfied ++ */ ++ usable_nodes--; ++ if (usable_nodes && required_kernelcore > usable_nodes) ++ goto restart; ++ ++out2: ++ /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ ++ for (nid = 0; nid < MAX_NUMNODES; nid++) ++ zone_movable_pfn[nid] = ++ roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); ++ ++out: ++ /* restore the node_state */ ++ node_states[N_MEMORY] = saved_node_state; ++} ++ ++/* Any regular or high memory on that node ? */ ++static void check_for_memory(pg_data_t *pgdat, int nid) ++{ ++ enum zone_type zone_type; ++ ++ if (N_MEMORY == N_NORMAL_MEMORY) ++ return; ++ ++ for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { ++ struct zone *zone = &pgdat->node_zones[zone_type]; ++ if (populated_zone(zone)) { ++ node_set_state(nid, N_HIGH_MEMORY); ++ if (N_NORMAL_MEMORY != N_HIGH_MEMORY && ++ zone_type <= ZONE_NORMAL) ++ node_set_state(nid, N_NORMAL_MEMORY); ++ break; ++ } ++ } ++} ++ ++/** ++ * free_area_init_nodes - Initialise all pg_data_t and zone data ++ * @max_zone_pfn: an array of max PFNs for each zone ++ * ++ * This will call free_area_init_node() for each active node in the system. ++ * Using the page ranges provided by memblock_set_node(), the size of each ++ * zone in each node and their holes is calculated. If the maximum PFN ++ * between two adjacent zones match, it is assumed that the zone is empty. ++ * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed ++ * that arch_max_dma32_pfn has no pages. It is also assumed that a zone ++ * starts where the previous one ended. For example, ZONE_DMA32 starts ++ * at arch_max_dma_pfn. ++ */ ++void __init free_area_init_nodes(unsigned long *max_zone_pfn) ++{ ++ unsigned long start_pfn, end_pfn; ++ int i, nid; ++ ++ /* Record where the zone boundaries are */ ++ memset(arch_zone_lowest_possible_pfn, 0, ++ sizeof(arch_zone_lowest_possible_pfn)); ++ memset(arch_zone_highest_possible_pfn, 0, ++ sizeof(arch_zone_highest_possible_pfn)); ++ arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions(); ++ arch_zone_highest_possible_pfn[0] = max_zone_pfn[0]; ++ for (i = 1; i < MAX_NR_ZONES; i++) { ++ if (i == ZONE_MOVABLE) ++ continue; ++ arch_zone_lowest_possible_pfn[i] = ++ arch_zone_highest_possible_pfn[i-1]; ++ arch_zone_highest_possible_pfn[i] = ++ max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]); ++ } ++ arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0; ++ arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0; ++ ++ /* Find the PFNs that ZONE_MOVABLE begins at in each node */ ++ memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); ++ find_zone_movable_pfns_for_nodes(); ++ ++ /* Print out the zone ranges */ ++ pr_info("Zone ranges:\n"); ++ for (i = 0; i < MAX_NR_ZONES; i++) { ++ if (i == ZONE_MOVABLE) ++ continue; ++ pr_info(" %-8s ", zone_names[i]); ++ if (arch_zone_lowest_possible_pfn[i] == ++ arch_zone_highest_possible_pfn[i]) ++ pr_cont("empty\n"); ++ else ++ pr_cont("[mem %#018Lx-%#018Lx]\n", ++ (u64)arch_zone_lowest_possible_pfn[i] ++ << PAGE_SHIFT, ++ ((u64)arch_zone_highest_possible_pfn[i] ++ << PAGE_SHIFT) - 1); ++ } ++ ++ /* Print out the PFNs ZONE_MOVABLE begins at in each node */ ++ pr_info("Movable zone start for each node\n"); ++ for (i = 0; i < MAX_NUMNODES; i++) { ++ if (zone_movable_pfn[i]) ++ pr_info(" Node %d: %#018Lx\n", i, ++ (u64)zone_movable_pfn[i] << PAGE_SHIFT); ++ } ++ ++ /* Print out the early node map */ ++ pr_info("Early memory node ranges\n"); ++ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) ++ pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, ++ (u64)start_pfn << PAGE_SHIFT, ++ ((u64)end_pfn << PAGE_SHIFT) - 1); ++ ++ /* Initialise every node */ ++ mminit_verify_pageflags_layout(); ++ setup_nr_node_ids(); ++ for_each_online_node(nid) { ++ pg_data_t *pgdat = NODE_DATA(nid); ++ free_area_init_node(nid, NULL, ++ find_min_pfn_for_node(nid), NULL); ++ ++ /* Any memory on that node */ ++ if (pgdat->node_present_pages) ++ node_set_state(nid, N_MEMORY); ++ check_for_memory(pgdat, nid); ++ } ++} ++ ++static int __init cmdline_parse_core(char *p, unsigned long *core) ++{ ++ unsigned long long coremem; ++ if (!p) ++ return -EINVAL; ++ ++ coremem = memparse(p, &p); ++ *core = coremem >> PAGE_SHIFT; ++ ++ /* Paranoid check that UL is enough for the coremem value */ ++ WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); ++ ++ return 0; ++} ++ ++/* ++ * kernelcore=size sets the amount of memory for use for allocations that ++ * cannot be reclaimed or migrated. ++ */ ++static int __init cmdline_parse_kernelcore(char *p) ++{ ++ return cmdline_parse_core(p, &required_kernelcore); ++} ++ ++/* ++ * movablecore=size sets the amount of memory for use for allocations that ++ * can be reclaimed or migrated. ++ */ ++static int __init cmdline_parse_movablecore(char *p) ++{ ++ return cmdline_parse_core(p, &required_movablecore); ++} ++ ++early_param("kernelcore", cmdline_parse_kernelcore); ++early_param("movablecore", cmdline_parse_movablecore); ++ ++#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++ ++void adjust_managed_page_count(struct page *page, long count) ++{ ++ spin_lock(&managed_page_count_lock); ++ page_zone(page)->managed_pages += count; ++ totalram_pages += count; ++#ifdef CONFIG_HIGHMEM ++ if (PageHighMem(page)) ++ totalhigh_pages += count; ++#endif ++ spin_unlock(&managed_page_count_lock); ++} ++EXPORT_SYMBOL(adjust_managed_page_count); ++ ++unsigned long free_reserved_area(void *start, void *end, int poison, char *s) ++{ ++ void *pos; ++ unsigned long pages = 0; ++ ++ start = (void *)PAGE_ALIGN((unsigned long)start); ++ end = (void *)((unsigned long)end & PAGE_MASK); ++ for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { ++ if ((unsigned int)poison <= 0xFF) ++ memset(pos, poison, PAGE_SIZE); ++ free_reserved_page(virt_to_page(pos)); ++ } ++ ++ if (pages && s) ++ pr_info("Freeing %s memory: %ldK (%p - %p)\n", ++ s, pages << (PAGE_SHIFT - 10), start, end); ++ ++ return pages; ++} ++EXPORT_SYMBOL(free_reserved_area); ++ ++#ifdef CONFIG_HIGHMEM ++void free_highmem_page(struct page *page) ++{ ++ __free_reserved_page(page); ++ totalram_pages++; ++ page_zone(page)->managed_pages++; ++ totalhigh_pages++; ++} ++#endif ++ ++ ++void __init mem_init_print_info(const char *str) ++{ ++ unsigned long physpages, codesize, datasize, rosize, bss_size; ++ unsigned long init_code_size, init_data_size; ++ ++ physpages = get_num_physpages(); ++ codesize = _etext - _stext; ++ datasize = _edata - _sdata; ++ rosize = __end_rodata - __start_rodata; ++ bss_size = __bss_stop - __bss_start; ++ init_data_size = __init_end - __init_begin; ++ init_code_size = _einittext - _sinittext; ++ ++ /* ++ * Detect special cases and adjust section sizes accordingly: ++ * 1) .init.* may be embedded into .data sections ++ * 2) .init.text.* may be out of [__init_begin, __init_end], ++ * please refer to arch/tile/kernel/vmlinux.lds.S. ++ * 3) .rodata.* may be embedded into .text or .data sections. ++ */ ++#define adj_init_size(start, end, size, pos, adj) \ ++ do { \ ++ if (start <= pos && pos < end && size > adj) \ ++ size -= adj; \ ++ } while (0) ++ ++ adj_init_size(__init_begin, __init_end, init_data_size, ++ _sinittext, init_code_size); ++ adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); ++ adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); ++ adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); ++ adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); ++ ++#undef adj_init_size ++ ++ pr_info("Memory: %luK/%luK available " ++ "(%luK kernel code, %luK rwdata, %luK rodata, " ++ "%luK init, %luK bss, %luK reserved, %luK cma-reserved" ++#ifdef CONFIG_HIGHMEM ++ ", %luK highmem" ++#endif ++ "%s%s)\n", ++ nr_free_pages() << (PAGE_SHIFT-10), physpages << (PAGE_SHIFT-10), ++ codesize >> 10, datasize >> 10, rosize >> 10, ++ (init_data_size + init_code_size) >> 10, bss_size >> 10, ++ (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT-10), ++ totalcma_pages << (PAGE_SHIFT-10), ++#ifdef CONFIG_HIGHMEM ++ totalhigh_pages << (PAGE_SHIFT-10), ++#endif ++ str ? ", " : "", str ? str : ""); ++} ++ ++/** ++ * set_dma_reserve - set the specified number of pages reserved in the first zone ++ * @new_dma_reserve: The number of pages to mark reserved ++ * ++ * The per-cpu batchsize and zone watermarks are determined by present_pages. ++ * In the DMA zone, a significant percentage may be consumed by kernel image ++ * and other unfreeable allocations which can skew the watermarks badly. This ++ * function may optionally be used to account for unfreeable pages in the ++ * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and ++ * smaller per-cpu batchsize. ++ */ ++void __init set_dma_reserve(unsigned long new_dma_reserve) ++{ ++ dma_reserve = new_dma_reserve; ++} ++ ++void __init free_area_init(unsigned long *zones_size) ++{ ++ free_area_init_node(0, zones_size, ++ __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); ++} ++ ++static int page_alloc_cpu_notify(struct notifier_block *self, ++ unsigned long action, void *hcpu) ++{ ++ int cpu = (unsigned long)hcpu; ++ ++ if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { ++ lru_add_drain_cpu(cpu); ++ drain_pages(cpu); ++ ++ /* ++ * Spill the event counters of the dead processor ++ * into the current processors event counters. ++ * This artificially elevates the count of the current ++ * processor. ++ */ ++ vm_events_fold_cpu(cpu); ++ ++ /* ++ * Zero the differential counters of the dead processor ++ * so that the vm statistics are consistent. ++ * ++ * This is only okay since the processor is dead and cannot ++ * race with what we are doing. ++ */ ++ cpu_vm_stats_fold(cpu); ++ } ++ return NOTIFY_OK; ++} ++ ++void __init page_alloc_init(void) ++{ ++ hotcpu_notifier(page_alloc_cpu_notify, 0); ++} ++ ++/* ++ * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio ++ * or min_free_kbytes changes. ++ */ ++static void calculate_totalreserve_pages(void) ++{ ++ struct pglist_data *pgdat; ++ unsigned long reserve_pages = 0; ++ enum zone_type i, j; ++ ++ for_each_online_pgdat(pgdat) { ++ for (i = 0; i < MAX_NR_ZONES; i++) { ++ struct zone *zone = pgdat->node_zones + i; ++ long max = 0; ++ ++ /* Find valid and maximum lowmem_reserve in the zone */ ++ for (j = i; j < MAX_NR_ZONES; j++) { ++ if (zone->lowmem_reserve[j] > max) ++ max = zone->lowmem_reserve[j]; ++ } ++ ++ /* we treat the high watermark as reserved pages. */ ++ max += high_wmark_pages(zone); ++ ++ if (max > zone->managed_pages) ++ max = zone->managed_pages; ++ reserve_pages += max; ++ /* ++ * Lowmem reserves are not available to ++ * GFP_HIGHUSER page cache allocations and ++ * kswapd tries to balance zones to their high ++ * watermark. As a result, neither should be ++ * regarded as dirtyable memory, to prevent a ++ * situation where reclaim has to clean pages ++ * in order to balance the zones. ++ */ ++ zone->dirty_balance_reserve = max; ++ } ++ } ++ dirty_balance_reserve = reserve_pages; ++ totalreserve_pages = reserve_pages; ++} ++ ++/* ++ * setup_per_zone_lowmem_reserve - called whenever ++ * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone ++ * has a correct pages reserved value, so an adequate number of ++ * pages are left in the zone after a successful __alloc_pages(). ++ */ ++static void setup_per_zone_lowmem_reserve(void) ++{ ++ struct pglist_data *pgdat; ++ enum zone_type j, idx; ++ ++ for_each_online_pgdat(pgdat) { ++ for (j = 0; j < MAX_NR_ZONES; j++) { ++ struct zone *zone = pgdat->node_zones + j; ++ unsigned long managed_pages = zone->managed_pages; ++ ++ zone->lowmem_reserve[j] = 0; ++ ++ idx = j; ++ while (idx) { ++ struct zone *lower_zone; ++ ++ idx--; ++ ++ if (sysctl_lowmem_reserve_ratio[idx] < 1) ++ sysctl_lowmem_reserve_ratio[idx] = 1; ++ ++ lower_zone = pgdat->node_zones + idx; ++ lower_zone->lowmem_reserve[j] = managed_pages / ++ sysctl_lowmem_reserve_ratio[idx]; ++ managed_pages += lower_zone->managed_pages; ++ } ++ } ++ } ++ ++ /* update totalreserve_pages */ ++ calculate_totalreserve_pages(); ++} ++ ++static void __setup_per_zone_wmarks(void) ++{ ++ unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); ++ unsigned long lowmem_pages = 0; ++ struct zone *zone; ++ unsigned long flags; ++ ++ /* Calculate total number of !ZONE_HIGHMEM pages */ ++ for_each_zone(zone) { ++ if (!is_highmem(zone)) ++ lowmem_pages += zone->managed_pages; ++ } ++ ++ for_each_zone(zone) { ++ u64 tmp; ++ ++ spin_lock_irqsave(&zone->lock, flags); ++ tmp = (u64)pages_min * zone->managed_pages; ++ do_div(tmp, lowmem_pages); ++ if (is_highmem(zone)) { ++ /* ++ * __GFP_HIGH and PF_MEMALLOC allocations usually don't ++ * need highmem pages, so cap pages_min to a small ++ * value here. ++ * ++ * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) ++ * deltas control asynch page reclaim, and so should ++ * not be capped for highmem. ++ */ ++ unsigned long min_pages; ++ ++ min_pages = zone->managed_pages / 1024; ++ min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); ++ zone->watermark[WMARK_MIN] = min_pages; ++ } else { ++ /* ++ * If it's a lowmem zone, reserve a number of pages ++ * proportionate to the zone's size. ++ */ ++ zone->watermark[WMARK_MIN] = tmp; ++ } ++ ++ zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2); ++ zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1); ++ ++ __mod_zone_page_state(zone, NR_ALLOC_BATCH, ++ high_wmark_pages(zone) - low_wmark_pages(zone) - ++ atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); ++ ++ setup_zone_migrate_reserve(zone); ++ spin_unlock_irqrestore(&zone->lock, flags); ++ } ++ ++ /* update totalreserve_pages */ ++ calculate_totalreserve_pages(); ++} ++ ++/** ++ * setup_per_zone_wmarks - called when min_free_kbytes changes ++ * or when memory is hot-{added|removed} ++ * ++ * Ensures that the watermark[min,low,high] values for each zone are set ++ * correctly with respect to min_free_kbytes. ++ */ ++void setup_per_zone_wmarks(void) ++{ ++ mutex_lock(&zonelists_mutex); ++ __setup_per_zone_wmarks(); ++ mutex_unlock(&zonelists_mutex); ++} ++ ++/* ++ * The inactive anon list should be small enough that the VM never has to ++ * do too much work, but large enough that each inactive page has a chance ++ * to be referenced again before it is swapped out. ++ * ++ * The inactive_anon ratio is the target ratio of ACTIVE_ANON to ++ * INACTIVE_ANON pages on this zone's LRU, maintained by the ++ * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of ++ * the anonymous pages are kept on the inactive list. ++ * ++ * total target max ++ * memory ratio inactive anon ++ * ------------------------------------- ++ * 10MB 1 5MB ++ * 100MB 1 50MB ++ * 1GB 3 250MB ++ * 10GB 10 0.9GB ++ * 100GB 31 3GB ++ * 1TB 101 10GB ++ * 10TB 320 32GB ++ */ ++static void __meminit calculate_zone_inactive_ratio(struct zone *zone) ++{ ++ unsigned int gb, ratio; ++ ++ /* Zone size in gigabytes */ ++ gb = zone->managed_pages >> (30 - PAGE_SHIFT); ++ if (gb) ++ ratio = int_sqrt(10 * gb); ++ else ++ ratio = 1; ++ ++ zone->inactive_ratio = ratio; ++} ++ ++static void __meminit setup_per_zone_inactive_ratio(void) ++{ ++ struct zone *zone; ++ ++ for_each_zone(zone) ++ calculate_zone_inactive_ratio(zone); ++} ++ ++/* ++ * Initialise min_free_kbytes. ++ * ++ * For small machines we want it small (128k min). For large machines ++ * we want it large (64MB max). But it is not linear, because network ++ * bandwidth does not increase linearly with machine size. We use ++ * ++ * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: ++ * min_free_kbytes = sqrt(lowmem_kbytes * 16) ++ * ++ * which yields ++ * ++ * 16MB: 512k ++ * 32MB: 724k ++ * 64MB: 1024k ++ * 128MB: 1448k ++ * 256MB: 2048k ++ * 512MB: 2896k ++ * 1024MB: 4096k ++ * 2048MB: 5792k ++ * 4096MB: 8192k ++ * 8192MB: 11584k ++ * 16384MB: 16384k ++ */ ++int __meminit init_per_zone_wmark_min(void) ++{ ++ unsigned long lowmem_kbytes; ++ int new_min_free_kbytes; ++ ++ lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); ++ new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); ++ ++ if (new_min_free_kbytes > user_min_free_kbytes) { ++ min_free_kbytes = new_min_free_kbytes; ++ if (min_free_kbytes < 128) ++ min_free_kbytes = 128; ++ if (min_free_kbytes > 65536) ++ min_free_kbytes = 65536; ++ } else { ++ pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", ++ new_min_free_kbytes, user_min_free_kbytes); ++ } ++ setup_per_zone_wmarks(); ++ refresh_zone_stat_thresholds(); ++ setup_per_zone_lowmem_reserve(); ++ setup_per_zone_inactive_ratio(); ++ return 0; ++} ++module_init(init_per_zone_wmark_min) ++ ++/* ++ * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so ++ * that we can call two helper functions whenever min_free_kbytes ++ * changes. ++ */ ++int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ int rc; ++ ++ rc = proc_dointvec_minmax(table, write, buffer, length, ppos); ++ if (rc) ++ return rc; ++ ++ if (write) { ++ user_min_free_kbytes = min_free_kbytes; ++ setup_per_zone_wmarks(); ++ } ++ return 0; ++} ++ ++#ifdef CONFIG_NUMA ++int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ struct zone *zone; ++ int rc; ++ ++ rc = proc_dointvec_minmax(table, write, buffer, length, ppos); ++ if (rc) ++ return rc; ++ ++ for_each_zone(zone) ++ zone->min_unmapped_pages = (zone->managed_pages * ++ sysctl_min_unmapped_ratio) / 100; ++ return 0; ++} ++ ++int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ struct zone *zone; ++ int rc; ++ ++ rc = proc_dointvec_minmax(table, write, buffer, length, ppos); ++ if (rc) ++ return rc; ++ ++ for_each_zone(zone) ++ zone->min_slab_pages = (zone->managed_pages * ++ sysctl_min_slab_ratio) / 100; ++ return 0; ++} ++#endif ++ ++/* ++ * lowmem_reserve_ratio_sysctl_handler - just a wrapper around ++ * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() ++ * whenever sysctl_lowmem_reserve_ratio changes. ++ * ++ * The reserve ratio obviously has absolutely no relation with the ++ * minimum watermarks. The lowmem reserve ratio can only make sense ++ * if in function of the boot time zone sizes. ++ */ ++int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ proc_dointvec_minmax(table, write, buffer, length, ppos); ++ setup_per_zone_lowmem_reserve(); ++ return 0; ++} ++ ++/* ++ * percpu_pagelist_fraction - changes the pcp->high for each zone on each ++ * cpu. It is the fraction of total pages in each zone that a hot per cpu ++ * pagelist can have before it gets flushed back to buddy allocator. ++ */ ++int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ struct zone *zone; ++ int old_percpu_pagelist_fraction; ++ int ret; ++ ++ mutex_lock(&pcp_batch_high_lock); ++ old_percpu_pagelist_fraction = percpu_pagelist_fraction; ++ ++ ret = proc_dointvec_minmax(table, write, buffer, length, ppos); ++ if (!write || ret < 0) ++ goto out; ++ ++ /* Sanity checking to avoid pcp imbalance */ ++ if (percpu_pagelist_fraction && ++ percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { ++ percpu_pagelist_fraction = old_percpu_pagelist_fraction; ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ /* No change? */ ++ if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) ++ goto out; ++ ++ for_each_populated_zone(zone) { ++ unsigned int cpu; ++ ++ for_each_possible_cpu(cpu) ++ pageset_set_high_and_batch(zone, ++ per_cpu_ptr(zone->pageset, cpu)); ++ } ++out: ++ mutex_unlock(&pcp_batch_high_lock); ++ return ret; ++} ++ ++int hashdist = HASHDIST_DEFAULT; ++ ++#ifdef CONFIG_NUMA ++static int __init set_hashdist(char *str) ++{ ++ if (!str) ++ return 0; ++ hashdist = simple_strtoul(str, &str, 0); ++ return 1; ++} ++__setup("hashdist=", set_hashdist); ++#endif ++ ++/* ++ * allocate a large system hash table from bootmem ++ * - it is assumed that the hash table must contain an exact power-of-2 ++ * quantity of entries ++ * - limit is the number of hash buckets, not the total allocation size ++ */ ++void *__init alloc_large_system_hash(const char *tablename, ++ unsigned long bucketsize, ++ unsigned long numentries, ++ int scale, ++ int flags, ++ unsigned int *_hash_shift, ++ unsigned int *_hash_mask, ++ unsigned long low_limit, ++ unsigned long high_limit) ++{ ++ unsigned long long max = high_limit; ++ unsigned long log2qty, size; ++ void *table = NULL; ++ ++ /* allow the kernel cmdline to have a say */ ++ if (!numentries) { ++ /* round applicable memory size up to nearest megabyte */ ++ numentries = nr_kernel_pages; ++ ++ /* It isn't necessary when PAGE_SIZE >= 1MB */ ++ if (PAGE_SHIFT < 20) ++ numentries = round_up(numentries, (1<<20)/PAGE_SIZE); ++ ++ /* limit to 1 bucket per 2^scale bytes of low memory */ ++ if (scale > PAGE_SHIFT) ++ numentries >>= (scale - PAGE_SHIFT); ++ else ++ numentries <<= (PAGE_SHIFT - scale); ++ ++ /* Make sure we've got at least a 0-order allocation.. */ ++ if (unlikely(flags & HASH_SMALL)) { ++ /* Makes no sense without HASH_EARLY */ ++ WARN_ON(!(flags & HASH_EARLY)); ++ if (!(numentries >> *_hash_shift)) { ++ numentries = 1UL << *_hash_shift; ++ BUG_ON(!numentries); ++ } ++ } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) ++ numentries = PAGE_SIZE / bucketsize; ++ } ++ numentries = roundup_pow_of_two(numentries); ++ ++ /* limit allocation size to 1/16 total memory by default */ ++ if (max == 0) { ++ max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; ++ do_div(max, bucketsize); ++ } ++ max = min(max, 0x80000000ULL); ++ ++ if (numentries < low_limit) ++ numentries = low_limit; ++ if (numentries > max) ++ numentries = max; ++ ++ log2qty = ilog2(numentries); ++ ++ do { ++ size = bucketsize << log2qty; ++ if (flags & HASH_EARLY) ++ table = memblock_virt_alloc_nopanic(size, 0); ++ else if (hashdist) ++ table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); ++ else { ++ /* ++ * If bucketsize is not a power-of-two, we may free ++ * some pages at the end of hash table which ++ * alloc_pages_exact() automatically does ++ */ ++ if (get_order(size) < MAX_ORDER) { ++ table = alloc_pages_exact(size, GFP_ATOMIC); ++ kmemleak_alloc(table, size, 1, GFP_ATOMIC); ++ } ++ } ++ } while (!table && size > PAGE_SIZE && --log2qty); ++ ++ if (!table) ++ panic("Failed to allocate %s hash table\n", tablename); ++ ++ printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n", ++ tablename, ++ (1UL << log2qty), ++ ilog2(size) - PAGE_SHIFT, ++ size); ++ ++ if (_hash_shift) ++ *_hash_shift = log2qty; ++ if (_hash_mask) ++ *_hash_mask = (1 << log2qty) - 1; ++ ++ return table; ++} ++ ++/* Return a pointer to the bitmap storing bits affecting a block of pages */ ++static inline unsigned long *get_pageblock_bitmap(struct zone *zone, ++ unsigned long pfn) ++{ ++#ifdef CONFIG_SPARSEMEM ++ return __pfn_to_section(pfn)->pageblock_flags; ++#else ++ return zone->pageblock_flags; ++#endif /* CONFIG_SPARSEMEM */ ++} ++ ++static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn) ++{ ++#ifdef CONFIG_SPARSEMEM ++ pfn &= (PAGES_PER_SECTION-1); ++ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; ++#else ++ pfn = pfn - round_down(zone->zone_start_pfn, pageblock_nr_pages); ++ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; ++#endif /* CONFIG_SPARSEMEM */ ++} ++ ++/** ++ * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages ++ * @page: The page within the block of interest ++ * @pfn: The target page frame number ++ * @end_bitidx: The last bit of interest to retrieve ++ * @mask: mask of bits that the caller is interested in ++ * ++ * Return: pageblock_bits flags ++ */ ++unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, ++ unsigned long end_bitidx, ++ unsigned long mask) ++{ ++ struct zone *zone; ++ unsigned long *bitmap; ++ unsigned long bitidx, word_bitidx; ++ unsigned long word; ++ ++ zone = page_zone(page); ++ bitmap = get_pageblock_bitmap(zone, pfn); ++ bitidx = pfn_to_bitidx(zone, pfn); ++ word_bitidx = bitidx / BITS_PER_LONG; ++ bitidx &= (BITS_PER_LONG-1); ++ ++ word = bitmap[word_bitidx]; ++ bitidx += end_bitidx; ++ return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; ++} ++ ++/** ++ * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages ++ * @page: The page within the block of interest ++ * @flags: The flags to set ++ * @pfn: The target page frame number ++ * @end_bitidx: The last bit of interest ++ * @mask: mask of bits that the caller is interested in ++ */ ++void set_pfnblock_flags_mask(struct page *page, unsigned long flags, ++ unsigned long pfn, ++ unsigned long end_bitidx, ++ unsigned long mask) ++{ ++ struct zone *zone; ++ unsigned long *bitmap; ++ unsigned long bitidx, word_bitidx; ++ unsigned long old_word, word; ++ ++ BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); ++ ++ zone = page_zone(page); ++ bitmap = get_pageblock_bitmap(zone, pfn); ++ bitidx = pfn_to_bitidx(zone, pfn); ++ word_bitidx = bitidx / BITS_PER_LONG; ++ bitidx &= (BITS_PER_LONG-1); ++ ++ VM_BUG_ON_PAGE(!zone_spans_pfn(zone, pfn), page); ++ ++ bitidx += end_bitidx; ++ mask <<= (BITS_PER_LONG - bitidx - 1); ++ flags <<= (BITS_PER_LONG - bitidx - 1); ++ ++ word = READ_ONCE(bitmap[word_bitidx]); ++ for (;;) { ++ old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); ++ if (word == old_word) ++ break; ++ word = old_word; ++ } ++} ++ ++/* ++ * This function checks whether pageblock includes unmovable pages or not. ++ * If @count is not zero, it is okay to include less @count unmovable pages ++ * ++ * PageLRU check without isolation or lru_lock could race so that ++ * MIGRATE_MOVABLE block might include unmovable pages. It means you can't ++ * expect this function should be exact. ++ */ ++bool has_unmovable_pages(struct zone *zone, struct page *page, int count, ++ bool skip_hwpoisoned_pages) ++{ ++ unsigned long pfn, iter, found; ++ int mt; ++ ++ /* ++ * For avoiding noise data, lru_add_drain_all() should be called ++ * If ZONE_MOVABLE, the zone never contains unmovable pages ++ */ ++ if (zone_idx(zone) == ZONE_MOVABLE) ++ return false; ++ mt = get_pageblock_migratetype(page); ++ if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt)) ++ return false; ++ ++ pfn = page_to_pfn(page); ++ for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) { ++ unsigned long check = pfn + iter; ++ ++ if (!pfn_valid_within(check)) ++ continue; ++ ++ page = pfn_to_page(check); ++ ++ /* ++ * Hugepages are not in LRU lists, but they're movable. ++ * We need not scan over tail pages bacause we don't ++ * handle each tail page individually in migration. ++ */ ++ if (PageHuge(page)) { ++ iter = round_up(iter + 1, 1<<compound_order(page)) - 1; ++ continue; ++ } ++ ++ /* ++ * We can't use page_count without pin a page ++ * because another CPU can free compound page. ++ * This check already skips compound tails of THP ++ * because their page->_count is zero at all time. ++ */ ++ if (!atomic_read(&page->_count)) { ++ if (PageBuddy(page)) ++ iter += (1 << page_order(page)) - 1; ++ continue; ++ } ++ ++ /* ++ * The HWPoisoned page may be not in buddy system, and ++ * page_count() is not 0. ++ */ ++ if (skip_hwpoisoned_pages && PageHWPoison(page)) ++ continue; ++ ++ if (!PageLRU(page)) ++ found++; ++ /* ++ * If there are RECLAIMABLE pages, we need to check ++ * it. But now, memory offline itself doesn't call ++ * shrink_node_slabs() and it still to be fixed. ++ */ ++ /* ++ * If the page is not RAM, page_count()should be 0. ++ * we don't need more check. This is an _used_ not-movable page. ++ * ++ * The problematic thing here is PG_reserved pages. PG_reserved ++ * is set to both of a memory hole page and a _used_ kernel ++ * page at boot. ++ */ ++ if (found > count) ++ return true; ++ } ++ return false; ++} ++ ++bool is_pageblock_removable_nolock(struct page *page) ++{ ++ struct zone *zone; ++ unsigned long pfn; ++ ++ /* ++ * We have to be careful here because we are iterating over memory ++ * sections which are not zone aware so we might end up outside of ++ * the zone but still within the section. ++ * We have to take care about the node as well. If the node is offline ++ * its NODE_DATA will be NULL - see page_zone. ++ */ ++ if (!node_online(page_to_nid(page))) ++ return false; ++ ++ zone = page_zone(page); ++ pfn = page_to_pfn(page); ++ if (!zone_spans_pfn(zone, pfn)) ++ return false; ++ ++ return !has_unmovable_pages(zone, page, 0, true); ++} ++ ++#ifdef CONFIG_CMA ++ ++static unsigned long pfn_max_align_down(unsigned long pfn) ++{ ++ return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, ++ pageblock_nr_pages) - 1); ++} ++ ++static unsigned long pfn_max_align_up(unsigned long pfn) ++{ ++ return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, ++ pageblock_nr_pages)); ++} ++ ++/* [start, end) must belong to a single zone. */ ++static int __alloc_contig_migrate_range(struct compact_control *cc, ++ unsigned long start, unsigned long end) ++{ ++ /* This function is based on compact_zone() from compaction.c. */ ++ unsigned long nr_reclaimed; ++ unsigned long pfn = start; ++ unsigned int tries = 0; ++ int ret = 0; ++ ++ migrate_prep(); ++ ++ while (pfn < end || !list_empty(&cc->migratepages)) { ++ if (fatal_signal_pending(current)) { ++ ret = -EINTR; ++ break; ++ } ++ ++ if (list_empty(&cc->migratepages)) { ++ cc->nr_migratepages = 0; ++ pfn = isolate_migratepages_range(cc, pfn, end); ++ if (!pfn) { ++ ret = -EINTR; ++ break; ++ } ++ tries = 0; ++ } else if (++tries == 5) { ++ ret = ret < 0 ? ret : -EBUSY; ++ break; ++ } ++ ++ nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, ++ &cc->migratepages); ++ cc->nr_migratepages -= nr_reclaimed; ++ ++ ret = migrate_pages(&cc->migratepages, alloc_migrate_target, ++ NULL, 0, cc->mode, MR_CMA); ++ } ++ if (ret < 0) { ++ putback_movable_pages(&cc->migratepages); ++ return ret; ++ } ++ return 0; ++} ++ ++/** ++ * alloc_contig_range() -- tries to allocate given range of pages ++ * @start: start PFN to allocate ++ * @end: one-past-the-last PFN to allocate ++ * @migratetype: migratetype of the underlaying pageblocks (either ++ * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks ++ * in range must have the same migratetype and it must ++ * be either of the two. ++ * ++ * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES ++ * aligned, however it's the caller's responsibility to guarantee that ++ * we are the only thread that changes migrate type of pageblocks the ++ * pages fall in. ++ * ++ * The PFN range must belong to a single zone. ++ * ++ * Returns zero on success or negative error code. On success all ++ * pages which PFN is in [start, end) are allocated for the caller and ++ * need to be freed with free_contig_range(). ++ */ ++int alloc_contig_range(unsigned long start, unsigned long end, ++ unsigned migratetype) ++{ ++ unsigned long outer_start, outer_end; ++ int ret = 0, order; ++ ++ struct compact_control cc = { ++ .nr_migratepages = 0, ++ .order = -1, ++ .zone = page_zone(pfn_to_page(start)), ++ .mode = MIGRATE_SYNC, ++ .ignore_skip_hint = true, ++ }; ++ INIT_LIST_HEAD(&cc.migratepages); ++ ++ /* ++ * What we do here is we mark all pageblocks in range as ++ * MIGRATE_ISOLATE. Because pageblock and max order pages may ++ * have different sizes, and due to the way page allocator ++ * work, we align the range to biggest of the two pages so ++ * that page allocator won't try to merge buddies from ++ * different pageblocks and change MIGRATE_ISOLATE to some ++ * other migration type. ++ * ++ * Once the pageblocks are marked as MIGRATE_ISOLATE, we ++ * migrate the pages from an unaligned range (ie. pages that ++ * we are interested in). This will put all the pages in ++ * range back to page allocator as MIGRATE_ISOLATE. ++ * ++ * When this is done, we take the pages in range from page ++ * allocator removing them from the buddy system. This way ++ * page allocator will never consider using them. ++ * ++ * This lets us mark the pageblocks back as ++ * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the ++ * aligned range but not in the unaligned, original range are ++ * put back to page allocator so that buddy can use them. ++ */ ++ ++ ret = start_isolate_page_range(pfn_max_align_down(start), ++ pfn_max_align_up(end), migratetype, ++ false); ++ if (ret) ++ return ret; ++ ++ ret = __alloc_contig_migrate_range(&cc, start, end); ++ if (ret) ++ goto done; ++ ++ /* ++ * Pages from [start, end) are within a MAX_ORDER_NR_PAGES ++ * aligned blocks that are marked as MIGRATE_ISOLATE. What's ++ * more, all pages in [start, end) are free in page allocator. ++ * What we are going to do is to allocate all pages from ++ * [start, end) (that is remove them from page allocator). ++ * ++ * The only problem is that pages at the beginning and at the ++ * end of interesting range may be not aligned with pages that ++ * page allocator holds, ie. they can be part of higher order ++ * pages. Because of this, we reserve the bigger range and ++ * once this is done free the pages we are not interested in. ++ * ++ * We don't have to hold zone->lock here because the pages are ++ * isolated thus they won't get removed from buddy. ++ */ ++ ++ lru_add_drain_all(); ++ drain_all_pages(cc.zone); ++ ++ order = 0; ++ outer_start = start; ++ while (!PageBuddy(pfn_to_page(outer_start))) { ++ if (++order >= MAX_ORDER) { ++ ret = -EBUSY; ++ goto done; ++ } ++ outer_start &= ~0UL << order; ++ } ++ ++ /* Make sure the range is really isolated. */ ++ if (test_pages_isolated(outer_start, end, false)) { ++ pr_info("%s: [%lx, %lx) PFNs busy\n", ++ __func__, outer_start, end); ++ ret = -EBUSY; ++ goto done; ++ } ++ ++ /* Grab isolated pages from freelists. */ ++ outer_end = isolate_freepages_range(&cc, outer_start, end); ++ if (!outer_end) { ++ ret = -EBUSY; ++ goto done; ++ } ++ ++ /* Free head and tail (if any) */ ++ if (start != outer_start) ++ free_contig_range(outer_start, start - outer_start); ++ if (end != outer_end) ++ free_contig_range(end, outer_end - end); ++ ++done: ++ undo_isolate_page_range(pfn_max_align_down(start), ++ pfn_max_align_up(end), migratetype); ++ return ret; ++} ++ ++void free_contig_range(unsigned long pfn, unsigned nr_pages) ++{ ++ unsigned int count = 0; ++ ++ for (; nr_pages--; pfn++) { ++ struct page *page = pfn_to_page(pfn); ++ ++ count += page_count(page) != 1; ++ __free_page(page); ++ } ++ WARN(count != 0, "%d pages are still in use!\n", count); ++} ++#endif ++ ++#ifdef CONFIG_MEMORY_HOTPLUG ++/* ++ * The zone indicated has a new number of managed_pages; batch sizes and percpu ++ * page high values need to be recalulated. ++ */ ++void __meminit zone_pcp_update(struct zone *zone) ++{ ++ unsigned cpu; ++ mutex_lock(&pcp_batch_high_lock); ++ for_each_possible_cpu(cpu) ++ pageset_set_high_and_batch(zone, ++ per_cpu_ptr(zone->pageset, cpu)); ++ mutex_unlock(&pcp_batch_high_lock); ++} ++#endif ++ ++void zone_pcp_reset(struct zone *zone) ++{ ++ unsigned long flags; ++ int cpu; ++ struct per_cpu_pageset *pset; ++ ++ /* avoid races with drain_pages() */ ++ local_irq_save(flags); ++ if (zone->pageset != &boot_pageset) { ++ for_each_online_cpu(cpu) { ++ pset = per_cpu_ptr(zone->pageset, cpu); ++ drain_zonestat(zone, pset); ++ } ++ free_percpu(zone->pageset); ++ zone->pageset = &boot_pageset; ++ } ++ local_irq_restore(flags); ++} ++ ++#ifdef CONFIG_MEMORY_HOTREMOVE ++/* ++ * All pages in the range must be isolated before calling this. ++ */ ++void ++__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) ++{ ++ struct page *page; ++ struct zone *zone; ++ unsigned int order, i; ++ unsigned long pfn; ++ unsigned long flags; ++ /* find the first valid pfn */ ++ for (pfn = start_pfn; pfn < end_pfn; pfn++) ++ if (pfn_valid(pfn)) ++ break; ++ if (pfn == end_pfn) ++ return; ++ zone = page_zone(pfn_to_page(pfn)); ++ spin_lock_irqsave(&zone->lock, flags); ++ pfn = start_pfn; ++ while (pfn < end_pfn) { ++ if (!pfn_valid(pfn)) { ++ pfn++; ++ continue; ++ } ++ page = pfn_to_page(pfn); ++ /* ++ * The HWPoisoned page may be not in buddy system, and ++ * page_count() is not 0. ++ */ ++ if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { ++ pfn++; ++ SetPageReserved(page); ++ continue; ++ } ++ ++ BUG_ON(page_count(page)); ++ BUG_ON(!PageBuddy(page)); ++ order = page_order(page); ++#ifdef CONFIG_DEBUG_VM ++ printk(KERN_INFO "remove from free list %lx %d %lx\n", ++ pfn, 1 << order, end_pfn); ++#endif ++ list_del(&page->lru); ++ rmv_page_order(page); ++ zone->free_area[order].nr_free--; ++ for (i = 0; i < (1 << order); i++) ++ SetPageReserved((page+i)); ++ pfn += (1 << order); ++ } ++ spin_unlock_irqrestore(&zone->lock, flags); ++} ++#endif ++ ++#ifdef CONFIG_MEMORY_FAILURE ++bool is_free_buddy_page(struct page *page) ++{ ++ struct zone *zone = page_zone(page); ++ unsigned long pfn = page_to_pfn(page); ++ unsigned long flags; ++ unsigned int order; ++ ++ spin_lock_irqsave(&zone->lock, flags); ++ for (order = 0; order < MAX_ORDER; order++) { ++ struct page *page_head = page - (pfn & ((1 << order) - 1)); ++ ++ if (PageBuddy(page_head) && page_order(page_head) >= order) ++ break; ++ } ++ spin_unlock_irqrestore(&zone->lock, flags); ++ ++ return order < MAX_ORDER; ++} ++#endif +diff -Nur linux-4.1.10.orig/mm/slab.h linux-4.1.10/mm/slab.h +--- linux-4.1.10.orig/mm/slab.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/slab.h 2015-10-07 18:00:08.000000000 +0200 @@ -330,7 +330,11 @@ * The slab lists for all objects. */ @@ -25179,9 +56094,9 @@ diff -Nur linux-4.1.6.orig/mm/slab.h linux-4.1.6/mm/slab.h #ifdef CONFIG_SLAB struct list_head slabs_partial; /* partial list first, better asm code */ -diff -Nur linux-4.1.6.orig/mm/slub.c linux-4.1.6/mm/slub.c ---- linux-4.1.6.orig/mm/slub.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/slub.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/mm/slub.c linux-4.1.10/mm/slub.c +--- linux-4.1.10.orig/mm/slub.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/slub.c 2015-10-07 18:00:08.000000000 +0200 @@ -1069,7 +1069,7 @@ { struct kmem_cache_node *n = get_node(s, page_to_nid(page)); @@ -25320,7 +56235,7 @@ diff -Nur linux-4.1.6.orig/mm/slub.c linux-4.1.6/mm/slub.c - inc_slabs_node(s, page_to_nid(page), page->objects); page->slab_cache = s; __SetPageSlab(page); - if (page->pfmemalloc) + if (page_is_pfmemalloc(page)) @@ -1448,10 +1430,34 @@ page->freelist = start; page->inuse = page->objects; @@ -25666,9 +56581,5357 @@ diff -Nur linux-4.1.6.orig/mm/slub.c linux-4.1.6/mm/slub.c } for (i = 0; i < t.count; i++) { -diff -Nur linux-4.1.6.orig/mm/swap.c linux-4.1.6/mm/swap.c ---- linux-4.1.6.orig/mm/swap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/swap.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/mm/slub.c.orig linux-4.1.10/mm/slub.c.orig +--- linux-4.1.10.orig/mm/slub.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/mm/slub.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,5344 @@ ++/* ++ * SLUB: A slab allocator that limits cache line use instead of queuing ++ * objects in per cpu and per node lists. ++ * ++ * The allocator synchronizes using per slab locks or atomic operatios ++ * and only uses a centralized lock to manage a pool of partial slabs. ++ * ++ * (C) 2007 SGI, Christoph Lameter ++ * (C) 2011 Linux Foundation, Christoph Lameter ++ */ ++ ++#include <linux/mm.h> ++#include <linux/swap.h> /* struct reclaim_state */ ++#include <linux/module.h> ++#include <linux/bit_spinlock.h> ++#include <linux/interrupt.h> ++#include <linux/bitops.h> ++#include <linux/slab.h> ++#include "slab.h" ++#include <linux/proc_fs.h> ++#include <linux/notifier.h> ++#include <linux/seq_file.h> ++#include <linux/kasan.h> ++#include <linux/kmemcheck.h> ++#include <linux/cpu.h> ++#include <linux/cpuset.h> ++#include <linux/mempolicy.h> ++#include <linux/ctype.h> ++#include <linux/debugobjects.h> ++#include <linux/kallsyms.h> ++#include <linux/memory.h> ++#include <linux/math64.h> ++#include <linux/fault-inject.h> ++#include <linux/stacktrace.h> ++#include <linux/prefetch.h> ++#include <linux/memcontrol.h> ++ ++#include <trace/events/kmem.h> ++ ++#include "internal.h" ++ ++/* ++ * Lock order: ++ * 1. slab_mutex (Global Mutex) ++ * 2. node->list_lock ++ * 3. slab_lock(page) (Only on some arches and for debugging) ++ * ++ * slab_mutex ++ * ++ * The role of the slab_mutex is to protect the list of all the slabs ++ * and to synchronize major metadata changes to slab cache structures. ++ * ++ * The slab_lock is only used for debugging and on arches that do not ++ * have the ability to do a cmpxchg_double. It only protects the second ++ * double word in the page struct. Meaning ++ * A. page->freelist -> List of object free in a page ++ * B. page->counters -> Counters of objects ++ * C. page->frozen -> frozen state ++ * ++ * If a slab is frozen then it is exempt from list management. It is not ++ * on any list. The processor that froze the slab is the one who can ++ * perform list operations on the page. Other processors may put objects ++ * onto the freelist but the processor that froze the slab is the only ++ * one that can retrieve the objects from the page's freelist. ++ * ++ * The list_lock protects the partial and full list on each node and ++ * the partial slab counter. If taken then no new slabs may be added or ++ * removed from the lists nor make the number of partial slabs be modified. ++ * (Note that the total number of slabs is an atomic value that may be ++ * modified without taking the list lock). ++ * ++ * The list_lock is a centralized lock and thus we avoid taking it as ++ * much as possible. As long as SLUB does not have to handle partial ++ * slabs, operations can continue without any centralized lock. F.e. ++ * allocating a long series of objects that fill up slabs does not require ++ * the list lock. ++ * Interrupts are disabled during allocation and deallocation in order to ++ * make the slab allocator safe to use in the context of an irq. In addition ++ * interrupts are disabled to ensure that the processor does not change ++ * while handling per_cpu slabs, due to kernel preemption. ++ * ++ * SLUB assigns one slab for allocation to each processor. ++ * Allocations only occur from these slabs called cpu slabs. ++ * ++ * Slabs with free elements are kept on a partial list and during regular ++ * operations no list for full slabs is used. If an object in a full slab is ++ * freed then the slab will show up again on the partial lists. ++ * We track full slabs for debugging purposes though because otherwise we ++ * cannot scan all objects. ++ * ++ * Slabs are freed when they become empty. Teardown and setup is ++ * minimal so we rely on the page allocators per cpu caches for ++ * fast frees and allocs. ++ * ++ * Overloading of page flags that are otherwise used for LRU management. ++ * ++ * PageActive The slab is frozen and exempt from list processing. ++ * This means that the slab is dedicated to a purpose ++ * such as satisfying allocations for a specific ++ * processor. Objects may be freed in the slab while ++ * it is frozen but slab_free will then skip the usual ++ * list operations. It is up to the processor holding ++ * the slab to integrate the slab into the slab lists ++ * when the slab is no longer needed. ++ * ++ * One use of this flag is to mark slabs that are ++ * used for allocations. Then such a slab becomes a cpu ++ * slab. The cpu slab may be equipped with an additional ++ * freelist that allows lockless access to ++ * free objects in addition to the regular freelist ++ * that requires the slab lock. ++ * ++ * PageError Slab requires special handling due to debug ++ * options set. This moves slab handling out of ++ * the fast path and disables lockless freelists. ++ */ ++ ++static inline int kmem_cache_debug(struct kmem_cache *s) ++{ ++#ifdef CONFIG_SLUB_DEBUG ++ return unlikely(s->flags & SLAB_DEBUG_FLAGS); ++#else ++ return 0; ++#endif ++} ++ ++static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) ++{ ++#ifdef CONFIG_SLUB_CPU_PARTIAL ++ return !kmem_cache_debug(s); ++#else ++ return false; ++#endif ++} ++ ++/* ++ * Issues still to be resolved: ++ * ++ * - Support PAGE_ALLOC_DEBUG. Should be easy to do. ++ * ++ * - Variable sizing of the per node arrays ++ */ ++ ++/* Enable to test recovery from slab corruption on boot */ ++#undef SLUB_RESILIENCY_TEST ++ ++/* Enable to log cmpxchg failures */ ++#undef SLUB_DEBUG_CMPXCHG ++ ++/* ++ * Mininum number of partial slabs. These will be left on the partial ++ * lists even if they are empty. kmem_cache_shrink may reclaim them. ++ */ ++#define MIN_PARTIAL 5 ++ ++/* ++ * Maximum number of desirable partial slabs. ++ * The existence of more partial slabs makes kmem_cache_shrink ++ * sort the partial list by the number of objects in use. ++ */ ++#define MAX_PARTIAL 10 ++ ++#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \ ++ SLAB_POISON | SLAB_STORE_USER) ++ ++/* ++ * Debugging flags that require metadata to be stored in the slab. These get ++ * disabled when slub_debug=O is used and a cache's min order increases with ++ * metadata. ++ */ ++#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) ++ ++#define OO_SHIFT 16 ++#define OO_MASK ((1 << OO_SHIFT) - 1) ++#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */ ++ ++/* Internal SLUB flags */ ++#define __OBJECT_POISON 0x80000000UL /* Poison object */ ++#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */ ++ ++#ifdef CONFIG_SMP ++static struct notifier_block slab_notifier; ++#endif ++ ++/* ++ * Tracking user of a slab. ++ */ ++#define TRACK_ADDRS_COUNT 16 ++struct track { ++ unsigned long addr; /* Called from address */ ++#ifdef CONFIG_STACKTRACE ++ unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */ ++#endif ++ int cpu; /* Was running on cpu */ ++ int pid; /* Pid context */ ++ unsigned long when; /* When did the operation occur */ ++}; ++ ++enum track_item { TRACK_ALLOC, TRACK_FREE }; ++ ++#ifdef CONFIG_SYSFS ++static int sysfs_slab_add(struct kmem_cache *); ++static int sysfs_slab_alias(struct kmem_cache *, const char *); ++static void memcg_propagate_slab_attrs(struct kmem_cache *s); ++#else ++static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } ++static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) ++ { return 0; } ++static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { } ++#endif ++ ++static inline void stat(const struct kmem_cache *s, enum stat_item si) ++{ ++#ifdef CONFIG_SLUB_STATS ++ /* ++ * The rmw is racy on a preemptible kernel but this is acceptable, so ++ * avoid this_cpu_add()'s irq-disable overhead. ++ */ ++ raw_cpu_inc(s->cpu_slab->stat[si]); ++#endif ++} ++ ++/******************************************************************** ++ * Core slab cache functions ++ *******************************************************************/ ++ ++/* Verify that a pointer has an address that is valid within a slab page */ ++static inline int check_valid_pointer(struct kmem_cache *s, ++ struct page *page, const void *object) ++{ ++ void *base; ++ ++ if (!object) ++ return 1; ++ ++ base = page_address(page); ++ if (object < base || object >= base + page->objects * s->size || ++ (object - base) % s->size) { ++ return 0; ++ } ++ ++ return 1; ++} ++ ++static inline void *get_freepointer(struct kmem_cache *s, void *object) ++{ ++ return *(void **)(object + s->offset); ++} ++ ++static void prefetch_freepointer(const struct kmem_cache *s, void *object) ++{ ++ prefetch(object + s->offset); ++} ++ ++static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) ++{ ++ void *p; ++ ++#ifdef CONFIG_DEBUG_PAGEALLOC ++ probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p)); ++#else ++ p = get_freepointer(s, object); ++#endif ++ return p; ++} ++ ++static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) ++{ ++ *(void **)(object + s->offset) = fp; ++} ++ ++/* Loop over all objects in a slab */ ++#define for_each_object(__p, __s, __addr, __objects) \ ++ for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ ++ __p += (__s)->size) ++ ++#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \ ++ for (__p = (__addr), __idx = 1; __idx <= __objects;\ ++ __p += (__s)->size, __idx++) ++ ++/* Determine object index from a given position */ ++static inline int slab_index(void *p, struct kmem_cache *s, void *addr) ++{ ++ return (p - addr) / s->size; ++} ++ ++static inline size_t slab_ksize(const struct kmem_cache *s) ++{ ++#ifdef CONFIG_SLUB_DEBUG ++ /* ++ * Debugging requires use of the padding between object ++ * and whatever may come after it. ++ */ ++ if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) ++ return s->object_size; ++ ++#endif ++ /* ++ * If we have the need to store the freelist pointer ++ * back there or track user information then we can ++ * only use the space before that information. ++ */ ++ if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER)) ++ return s->inuse; ++ /* ++ * Else we can use all the padding etc for the allocation ++ */ ++ return s->size; ++} ++ ++static inline int order_objects(int order, unsigned long size, int reserved) ++{ ++ return ((PAGE_SIZE << order) - reserved) / size; ++} ++ ++static inline struct kmem_cache_order_objects oo_make(int order, ++ unsigned long size, int reserved) ++{ ++ struct kmem_cache_order_objects x = { ++ (order << OO_SHIFT) + order_objects(order, size, reserved) ++ }; ++ ++ return x; ++} ++ ++static inline int oo_order(struct kmem_cache_order_objects x) ++{ ++ return x.x >> OO_SHIFT; ++} ++ ++static inline int oo_objects(struct kmem_cache_order_objects x) ++{ ++ return x.x & OO_MASK; ++} ++ ++/* ++ * Per slab locking using the pagelock ++ */ ++static __always_inline void slab_lock(struct page *page) ++{ ++ bit_spin_lock(PG_locked, &page->flags); ++} ++ ++static __always_inline void slab_unlock(struct page *page) ++{ ++ __bit_spin_unlock(PG_locked, &page->flags); ++} ++ ++static inline void set_page_slub_counters(struct page *page, unsigned long counters_new) ++{ ++ struct page tmp; ++ tmp.counters = counters_new; ++ /* ++ * page->counters can cover frozen/inuse/objects as well ++ * as page->_count. If we assign to ->counters directly ++ * we run the risk of losing updates to page->_count, so ++ * be careful and only assign to the fields we need. ++ */ ++ page->frozen = tmp.frozen; ++ page->inuse = tmp.inuse; ++ page->objects = tmp.objects; ++} ++ ++/* Interrupts must be disabled (for the fallback code to work right) */ ++static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, ++ void *freelist_old, unsigned long counters_old, ++ void *freelist_new, unsigned long counters_new, ++ const char *n) ++{ ++ VM_BUG_ON(!irqs_disabled()); ++#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ ++ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) ++ if (s->flags & __CMPXCHG_DOUBLE) { ++ if (cmpxchg_double(&page->freelist, &page->counters, ++ freelist_old, counters_old, ++ freelist_new, counters_new)) ++ return true; ++ } else ++#endif ++ { ++ slab_lock(page); ++ if (page->freelist == freelist_old && ++ page->counters == counters_old) { ++ page->freelist = freelist_new; ++ set_page_slub_counters(page, counters_new); ++ slab_unlock(page); ++ return true; ++ } ++ slab_unlock(page); ++ } ++ ++ cpu_relax(); ++ stat(s, CMPXCHG_DOUBLE_FAIL); ++ ++#ifdef SLUB_DEBUG_CMPXCHG ++ pr_info("%s %s: cmpxchg double redo ", n, s->name); ++#endif ++ ++ return false; ++} ++ ++static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, ++ void *freelist_old, unsigned long counters_old, ++ void *freelist_new, unsigned long counters_new, ++ const char *n) ++{ ++#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ ++ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) ++ if (s->flags & __CMPXCHG_DOUBLE) { ++ if (cmpxchg_double(&page->freelist, &page->counters, ++ freelist_old, counters_old, ++ freelist_new, counters_new)) ++ return true; ++ } else ++#endif ++ { ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ slab_lock(page); ++ if (page->freelist == freelist_old && ++ page->counters == counters_old) { ++ page->freelist = freelist_new; ++ set_page_slub_counters(page, counters_new); ++ slab_unlock(page); ++ local_irq_restore(flags); ++ return true; ++ } ++ slab_unlock(page); ++ local_irq_restore(flags); ++ } ++ ++ cpu_relax(); ++ stat(s, CMPXCHG_DOUBLE_FAIL); ++ ++#ifdef SLUB_DEBUG_CMPXCHG ++ pr_info("%s %s: cmpxchg double redo ", n, s->name); ++#endif ++ ++ return false; ++} ++ ++#ifdef CONFIG_SLUB_DEBUG ++/* ++ * Determine a map of object in use on a page. ++ * ++ * Node listlock must be held to guarantee that the page does ++ * not vanish from under us. ++ */ ++static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map) ++{ ++ void *p; ++ void *addr = page_address(page); ++ ++ for (p = page->freelist; p; p = get_freepointer(s, p)) ++ set_bit(slab_index(p, s, addr), map); ++} ++ ++/* ++ * Debug settings: ++ */ ++#ifdef CONFIG_SLUB_DEBUG_ON ++static int slub_debug = DEBUG_DEFAULT_FLAGS; ++#else ++static int slub_debug; ++#endif ++ ++static char *slub_debug_slabs; ++static int disable_higher_order_debug; ++ ++/* ++ * slub is about to manipulate internal object metadata. This memory lies ++ * outside the range of the allocated object, so accessing it would normally ++ * be reported by kasan as a bounds error. metadata_access_enable() is used ++ * to tell kasan that these accesses are OK. ++ */ ++static inline void metadata_access_enable(void) ++{ ++ kasan_disable_current(); ++} ++ ++static inline void metadata_access_disable(void) ++{ ++ kasan_enable_current(); ++} ++ ++/* ++ * Object debugging ++ */ ++static void print_section(char *text, u8 *addr, unsigned int length) ++{ ++ metadata_access_enable(); ++ print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr, ++ length, 1); ++ metadata_access_disable(); ++} ++ ++static struct track *get_track(struct kmem_cache *s, void *object, ++ enum track_item alloc) ++{ ++ struct track *p; ++ ++ if (s->offset) ++ p = object + s->offset + sizeof(void *); ++ else ++ p = object + s->inuse; ++ ++ return p + alloc; ++} ++ ++static void set_track(struct kmem_cache *s, void *object, ++ enum track_item alloc, unsigned long addr) ++{ ++ struct track *p = get_track(s, object, alloc); ++ ++ if (addr) { ++#ifdef CONFIG_STACKTRACE ++ struct stack_trace trace; ++ int i; ++ ++ trace.nr_entries = 0; ++ trace.max_entries = TRACK_ADDRS_COUNT; ++ trace.entries = p->addrs; ++ trace.skip = 3; ++ metadata_access_enable(); ++ save_stack_trace(&trace); ++ metadata_access_disable(); ++ ++ /* See rant in lockdep.c */ ++ if (trace.nr_entries != 0 && ++ trace.entries[trace.nr_entries - 1] == ULONG_MAX) ++ trace.nr_entries--; ++ ++ for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++) ++ p->addrs[i] = 0; ++#endif ++ p->addr = addr; ++ p->cpu = smp_processor_id(); ++ p->pid = current->pid; ++ p->when = jiffies; ++ } else ++ memset(p, 0, sizeof(struct track)); ++} ++ ++static void init_tracking(struct kmem_cache *s, void *object) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return; ++ ++ set_track(s, object, TRACK_FREE, 0UL); ++ set_track(s, object, TRACK_ALLOC, 0UL); ++} ++ ++static void print_track(const char *s, struct track *t) ++{ ++ if (!t->addr) ++ return; ++ ++ pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n", ++ s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid); ++#ifdef CONFIG_STACKTRACE ++ { ++ int i; ++ for (i = 0; i < TRACK_ADDRS_COUNT; i++) ++ if (t->addrs[i]) ++ pr_err("\t%pS\n", (void *)t->addrs[i]); ++ else ++ break; ++ } ++#endif ++} ++ ++static void print_tracking(struct kmem_cache *s, void *object) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return; ++ ++ print_track("Allocated", get_track(s, object, TRACK_ALLOC)); ++ print_track("Freed", get_track(s, object, TRACK_FREE)); ++} ++ ++static void print_page_info(struct page *page) ++{ ++ pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n", ++ page, page->objects, page->inuse, page->freelist, page->flags); ++ ++} ++ ++static void slab_bug(struct kmem_cache *s, char *fmt, ...) ++{ ++ struct va_format vaf; ++ va_list args; ++ ++ va_start(args, fmt); ++ vaf.fmt = fmt; ++ vaf.va = &args; ++ pr_err("=============================================================================\n"); ++ pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf); ++ pr_err("-----------------------------------------------------------------------------\n\n"); ++ ++ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); ++ va_end(args); ++} ++ ++static void slab_fix(struct kmem_cache *s, char *fmt, ...) ++{ ++ struct va_format vaf; ++ va_list args; ++ ++ va_start(args, fmt); ++ vaf.fmt = fmt; ++ vaf.va = &args; ++ pr_err("FIX %s: %pV\n", s->name, &vaf); ++ va_end(args); ++} ++ ++static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) ++{ ++ unsigned int off; /* Offset of last byte */ ++ u8 *addr = page_address(page); ++ ++ print_tracking(s, p); ++ ++ print_page_info(page); ++ ++ pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n", ++ p, p - addr, get_freepointer(s, p)); ++ ++ if (p > addr + 16) ++ print_section("Bytes b4 ", p - 16, 16); ++ ++ print_section("Object ", p, min_t(unsigned long, s->object_size, ++ PAGE_SIZE)); ++ if (s->flags & SLAB_RED_ZONE) ++ print_section("Redzone ", p + s->object_size, ++ s->inuse - s->object_size); ++ ++ if (s->offset) ++ off = s->offset + sizeof(void *); ++ else ++ off = s->inuse; ++ ++ if (s->flags & SLAB_STORE_USER) ++ off += 2 * sizeof(struct track); ++ ++ if (off != s->size) ++ /* Beginning of the filler is the free pointer */ ++ print_section("Padding ", p + off, s->size - off); ++ ++ dump_stack(); ++} ++ ++void object_err(struct kmem_cache *s, struct page *page, ++ u8 *object, char *reason) ++{ ++ slab_bug(s, "%s", reason); ++ print_trailer(s, page, object); ++} ++ ++static void slab_err(struct kmem_cache *s, struct page *page, ++ const char *fmt, ...) ++{ ++ va_list args; ++ char buf[100]; ++ ++ va_start(args, fmt); ++ vsnprintf(buf, sizeof(buf), fmt, args); ++ va_end(args); ++ slab_bug(s, "%s", buf); ++ print_page_info(page); ++ dump_stack(); ++} ++ ++static void init_object(struct kmem_cache *s, void *object, u8 val) ++{ ++ u8 *p = object; ++ ++ if (s->flags & __OBJECT_POISON) { ++ memset(p, POISON_FREE, s->object_size - 1); ++ p[s->object_size - 1] = POISON_END; ++ } ++ ++ if (s->flags & SLAB_RED_ZONE) ++ memset(p + s->object_size, val, s->inuse - s->object_size); ++} ++ ++static void restore_bytes(struct kmem_cache *s, char *message, u8 data, ++ void *from, void *to) ++{ ++ slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data); ++ memset(from, data, to - from); ++} ++ ++static int check_bytes_and_report(struct kmem_cache *s, struct page *page, ++ u8 *object, char *what, ++ u8 *start, unsigned int value, unsigned int bytes) ++{ ++ u8 *fault; ++ u8 *end; ++ ++ metadata_access_enable(); ++ fault = memchr_inv(start, value, bytes); ++ metadata_access_disable(); ++ if (!fault) ++ return 1; ++ ++ end = start + bytes; ++ while (end > fault && end[-1] == value) ++ end--; ++ ++ slab_bug(s, "%s overwritten", what); ++ pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n", ++ fault, end - 1, fault[0], value); ++ print_trailer(s, page, object); ++ ++ restore_bytes(s, what, value, fault, end); ++ return 0; ++} ++ ++/* ++ * Object layout: ++ * ++ * object address ++ * Bytes of the object to be managed. ++ * If the freepointer may overlay the object then the free ++ * pointer is the first word of the object. ++ * ++ * Poisoning uses 0x6b (POISON_FREE) and the last byte is ++ * 0xa5 (POISON_END) ++ * ++ * object + s->object_size ++ * Padding to reach word boundary. This is also used for Redzoning. ++ * Padding is extended by another word if Redzoning is enabled and ++ * object_size == inuse. ++ * ++ * We fill with 0xbb (RED_INACTIVE) for inactive objects and with ++ * 0xcc (RED_ACTIVE) for objects in use. ++ * ++ * object + s->inuse ++ * Meta data starts here. ++ * ++ * A. Free pointer (if we cannot overwrite object on free) ++ * B. Tracking data for SLAB_STORE_USER ++ * C. Padding to reach required alignment boundary or at mininum ++ * one word if debugging is on to be able to detect writes ++ * before the word boundary. ++ * ++ * Padding is done using 0x5a (POISON_INUSE) ++ * ++ * object + s->size ++ * Nothing is used beyond s->size. ++ * ++ * If slabcaches are merged then the object_size and inuse boundaries are mostly ++ * ignored. And therefore no slab options that rely on these boundaries ++ * may be used with merged slabcaches. ++ */ ++ ++static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) ++{ ++ unsigned long off = s->inuse; /* The end of info */ ++ ++ if (s->offset) ++ /* Freepointer is placed after the object. */ ++ off += sizeof(void *); ++ ++ if (s->flags & SLAB_STORE_USER) ++ /* We also have user information there */ ++ off += 2 * sizeof(struct track); ++ ++ if (s->size == off) ++ return 1; ++ ++ return check_bytes_and_report(s, page, p, "Object padding", ++ p + off, POISON_INUSE, s->size - off); ++} ++ ++/* Check the pad bytes at the end of a slab page */ ++static int slab_pad_check(struct kmem_cache *s, struct page *page) ++{ ++ u8 *start; ++ u8 *fault; ++ u8 *end; ++ int length; ++ int remainder; ++ ++ if (!(s->flags & SLAB_POISON)) ++ return 1; ++ ++ start = page_address(page); ++ length = (PAGE_SIZE << compound_order(page)) - s->reserved; ++ end = start + length; ++ remainder = length % s->size; ++ if (!remainder) ++ return 1; ++ ++ metadata_access_enable(); ++ fault = memchr_inv(end - remainder, POISON_INUSE, remainder); ++ metadata_access_disable(); ++ if (!fault) ++ return 1; ++ while (end > fault && end[-1] == POISON_INUSE) ++ end--; ++ ++ slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1); ++ print_section("Padding ", end - remainder, remainder); ++ ++ restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end); ++ return 0; ++} ++ ++static int check_object(struct kmem_cache *s, struct page *page, ++ void *object, u8 val) ++{ ++ u8 *p = object; ++ u8 *endobject = object + s->object_size; ++ ++ if (s->flags & SLAB_RED_ZONE) { ++ if (!check_bytes_and_report(s, page, object, "Redzone", ++ endobject, val, s->inuse - s->object_size)) ++ return 0; ++ } else { ++ if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { ++ check_bytes_and_report(s, page, p, "Alignment padding", ++ endobject, POISON_INUSE, ++ s->inuse - s->object_size); ++ } ++ } ++ ++ if (s->flags & SLAB_POISON) { ++ if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) && ++ (!check_bytes_and_report(s, page, p, "Poison", p, ++ POISON_FREE, s->object_size - 1) || ++ !check_bytes_and_report(s, page, p, "Poison", ++ p + s->object_size - 1, POISON_END, 1))) ++ return 0; ++ /* ++ * check_pad_bytes cleans up on its own. ++ */ ++ check_pad_bytes(s, page, p); ++ } ++ ++ if (!s->offset && val == SLUB_RED_ACTIVE) ++ /* ++ * Object and freepointer overlap. Cannot check ++ * freepointer while object is allocated. ++ */ ++ return 1; ++ ++ /* Check free pointer validity */ ++ if (!check_valid_pointer(s, page, get_freepointer(s, p))) { ++ object_err(s, page, p, "Freepointer corrupt"); ++ /* ++ * No choice but to zap it and thus lose the remainder ++ * of the free objects in this slab. May cause ++ * another error because the object count is now wrong. ++ */ ++ set_freepointer(s, p, NULL); ++ return 0; ++ } ++ return 1; ++} ++ ++static int check_slab(struct kmem_cache *s, struct page *page) ++{ ++ int maxobj; ++ ++ VM_BUG_ON(!irqs_disabled()); ++ ++ if (!PageSlab(page)) { ++ slab_err(s, page, "Not a valid slab page"); ++ return 0; ++ } ++ ++ maxobj = order_objects(compound_order(page), s->size, s->reserved); ++ if (page->objects > maxobj) { ++ slab_err(s, page, "objects %u > max %u", ++ page->objects, maxobj); ++ return 0; ++ } ++ if (page->inuse > page->objects) { ++ slab_err(s, page, "inuse %u > max %u", ++ page->inuse, page->objects); ++ return 0; ++ } ++ /* Slab_pad_check fixes things up after itself */ ++ slab_pad_check(s, page); ++ return 1; ++} ++ ++/* ++ * Determine if a certain object on a page is on the freelist. Must hold the ++ * slab lock to guarantee that the chains are in a consistent state. ++ */ ++static int on_freelist(struct kmem_cache *s, struct page *page, void *search) ++{ ++ int nr = 0; ++ void *fp; ++ void *object = NULL; ++ int max_objects; ++ ++ fp = page->freelist; ++ while (fp && nr <= page->objects) { ++ if (fp == search) ++ return 1; ++ if (!check_valid_pointer(s, page, fp)) { ++ if (object) { ++ object_err(s, page, object, ++ "Freechain corrupt"); ++ set_freepointer(s, object, NULL); ++ } else { ++ slab_err(s, page, "Freepointer corrupt"); ++ page->freelist = NULL; ++ page->inuse = page->objects; ++ slab_fix(s, "Freelist cleared"); ++ return 0; ++ } ++ break; ++ } ++ object = fp; ++ fp = get_freepointer(s, object); ++ nr++; ++ } ++ ++ max_objects = order_objects(compound_order(page), s->size, s->reserved); ++ if (max_objects > MAX_OBJS_PER_PAGE) ++ max_objects = MAX_OBJS_PER_PAGE; ++ ++ if (page->objects != max_objects) { ++ slab_err(s, page, "Wrong number of objects. Found %d but " ++ "should be %d", page->objects, max_objects); ++ page->objects = max_objects; ++ slab_fix(s, "Number of objects adjusted."); ++ } ++ if (page->inuse != page->objects - nr) { ++ slab_err(s, page, "Wrong object count. Counter is %d but " ++ "counted were %d", page->inuse, page->objects - nr); ++ page->inuse = page->objects - nr; ++ slab_fix(s, "Object count adjusted."); ++ } ++ return search == NULL; ++} ++ ++static void trace(struct kmem_cache *s, struct page *page, void *object, ++ int alloc) ++{ ++ if (s->flags & SLAB_TRACE) { ++ pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n", ++ s->name, ++ alloc ? "alloc" : "free", ++ object, page->inuse, ++ page->freelist); ++ ++ if (!alloc) ++ print_section("Object ", (void *)object, ++ s->object_size); ++ ++ dump_stack(); ++ } ++} ++ ++/* ++ * Tracking of fully allocated slabs for debugging purposes. ++ */ ++static void add_full(struct kmem_cache *s, ++ struct kmem_cache_node *n, struct page *page) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return; ++ ++ lockdep_assert_held(&n->list_lock); ++ list_add(&page->lru, &n->full); ++} ++ ++static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return; ++ ++ lockdep_assert_held(&n->list_lock); ++ list_del(&page->lru); ++} ++ ++/* Tracking of the number of slabs for debugging purposes */ ++static inline unsigned long slabs_node(struct kmem_cache *s, int node) ++{ ++ struct kmem_cache_node *n = get_node(s, node); ++ ++ return atomic_long_read(&n->nr_slabs); ++} ++ ++static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) ++{ ++ return atomic_long_read(&n->nr_slabs); ++} ++ ++static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) ++{ ++ struct kmem_cache_node *n = get_node(s, node); ++ ++ /* ++ * May be called early in order to allocate a slab for the ++ * kmem_cache_node structure. Solve the chicken-egg ++ * dilemma by deferring the increment of the count during ++ * bootstrap (see early_kmem_cache_node_alloc). ++ */ ++ if (likely(n)) { ++ atomic_long_inc(&n->nr_slabs); ++ atomic_long_add(objects, &n->total_objects); ++ } ++} ++static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) ++{ ++ struct kmem_cache_node *n = get_node(s, node); ++ ++ atomic_long_dec(&n->nr_slabs); ++ atomic_long_sub(objects, &n->total_objects); ++} ++ ++/* Object debug checks for alloc/free paths */ ++static void setup_object_debug(struct kmem_cache *s, struct page *page, ++ void *object) ++{ ++ if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))) ++ return; ++ ++ init_object(s, object, SLUB_RED_INACTIVE); ++ init_tracking(s, object); ++} ++ ++static noinline int alloc_debug_processing(struct kmem_cache *s, ++ struct page *page, ++ void *object, unsigned long addr) ++{ ++ if (!check_slab(s, page)) ++ goto bad; ++ ++ if (!check_valid_pointer(s, page, object)) { ++ object_err(s, page, object, "Freelist Pointer check fails"); ++ goto bad; ++ } ++ ++ if (!check_object(s, page, object, SLUB_RED_INACTIVE)) ++ goto bad; ++ ++ /* Success perform special debug activities for allocs */ ++ if (s->flags & SLAB_STORE_USER) ++ set_track(s, object, TRACK_ALLOC, addr); ++ trace(s, page, object, 1); ++ init_object(s, object, SLUB_RED_ACTIVE); ++ return 1; ++ ++bad: ++ if (PageSlab(page)) { ++ /* ++ * If this is a slab page then lets do the best we can ++ * to avoid issues in the future. Marking all objects ++ * as used avoids touching the remaining objects. ++ */ ++ slab_fix(s, "Marking all objects used"); ++ page->inuse = page->objects; ++ page->freelist = NULL; ++ } ++ return 0; ++} ++ ++static noinline struct kmem_cache_node *free_debug_processing( ++ struct kmem_cache *s, struct page *page, void *object, ++ unsigned long addr, unsigned long *flags) ++{ ++ struct kmem_cache_node *n = get_node(s, page_to_nid(page)); ++ ++ spin_lock_irqsave(&n->list_lock, *flags); ++ slab_lock(page); ++ ++ if (!check_slab(s, page)) ++ goto fail; ++ ++ if (!check_valid_pointer(s, page, object)) { ++ slab_err(s, page, "Invalid object pointer 0x%p", object); ++ goto fail; ++ } ++ ++ if (on_freelist(s, page, object)) { ++ object_err(s, page, object, "Object already free"); ++ goto fail; ++ } ++ ++ if (!check_object(s, page, object, SLUB_RED_ACTIVE)) ++ goto out; ++ ++ if (unlikely(s != page->slab_cache)) { ++ if (!PageSlab(page)) { ++ slab_err(s, page, "Attempt to free object(0x%p) " ++ "outside of slab", object); ++ } else if (!page->slab_cache) { ++ pr_err("SLUB <none>: no slab for object 0x%p.\n", ++ object); ++ dump_stack(); ++ } else ++ object_err(s, page, object, ++ "page slab pointer corrupt."); ++ goto fail; ++ } ++ ++ if (s->flags & SLAB_STORE_USER) ++ set_track(s, object, TRACK_FREE, addr); ++ trace(s, page, object, 0); ++ init_object(s, object, SLUB_RED_INACTIVE); ++out: ++ slab_unlock(page); ++ /* ++ * Keep node_lock to preserve integrity ++ * until the object is actually freed ++ */ ++ return n; ++ ++fail: ++ slab_unlock(page); ++ spin_unlock_irqrestore(&n->list_lock, *flags); ++ slab_fix(s, "Object at 0x%p not freed", object); ++ return NULL; ++} ++ ++static int __init setup_slub_debug(char *str) ++{ ++ slub_debug = DEBUG_DEFAULT_FLAGS; ++ if (*str++ != '=' || !*str) ++ /* ++ * No options specified. Switch on full debugging. ++ */ ++ goto out; ++ ++ if (*str == ',') ++ /* ++ * No options but restriction on slabs. This means full ++ * debugging for slabs matching a pattern. ++ */ ++ goto check_slabs; ++ ++ slub_debug = 0; ++ if (*str == '-') ++ /* ++ * Switch off all debugging measures. ++ */ ++ goto out; ++ ++ /* ++ * Determine which debug features should be switched on ++ */ ++ for (; *str && *str != ','; str++) { ++ switch (tolower(*str)) { ++ case 'f': ++ slub_debug |= SLAB_DEBUG_FREE; ++ break; ++ case 'z': ++ slub_debug |= SLAB_RED_ZONE; ++ break; ++ case 'p': ++ slub_debug |= SLAB_POISON; ++ break; ++ case 'u': ++ slub_debug |= SLAB_STORE_USER; ++ break; ++ case 't': ++ slub_debug |= SLAB_TRACE; ++ break; ++ case 'a': ++ slub_debug |= SLAB_FAILSLAB; ++ break; ++ case 'o': ++ /* ++ * Avoid enabling debugging on caches if its minimum ++ * order would increase as a result. ++ */ ++ disable_higher_order_debug = 1; ++ break; ++ default: ++ pr_err("slub_debug option '%c' unknown. skipped\n", ++ *str); ++ } ++ } ++ ++check_slabs: ++ if (*str == ',') ++ slub_debug_slabs = str + 1; ++out: ++ return 1; ++} ++ ++__setup("slub_debug", setup_slub_debug); ++ ++unsigned long kmem_cache_flags(unsigned long object_size, ++ unsigned long flags, const char *name, ++ void (*ctor)(void *)) ++{ ++ /* ++ * Enable debugging if selected on the kernel commandline. ++ */ ++ if (slub_debug && (!slub_debug_slabs || (name && ++ !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))) ++ flags |= slub_debug; ++ ++ return flags; ++} ++#else ++static inline void setup_object_debug(struct kmem_cache *s, ++ struct page *page, void *object) {} ++ ++static inline int alloc_debug_processing(struct kmem_cache *s, ++ struct page *page, void *object, unsigned long addr) { return 0; } ++ ++static inline struct kmem_cache_node *free_debug_processing( ++ struct kmem_cache *s, struct page *page, void *object, ++ unsigned long addr, unsigned long *flags) { return NULL; } ++ ++static inline int slab_pad_check(struct kmem_cache *s, struct page *page) ++ { return 1; } ++static inline int check_object(struct kmem_cache *s, struct page *page, ++ void *object, u8 val) { return 1; } ++static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, ++ struct page *page) {} ++static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, ++ struct page *page) {} ++unsigned long kmem_cache_flags(unsigned long object_size, ++ unsigned long flags, const char *name, ++ void (*ctor)(void *)) ++{ ++ return flags; ++} ++#define slub_debug 0 ++ ++#define disable_higher_order_debug 0 ++ ++static inline unsigned long slabs_node(struct kmem_cache *s, int node) ++ { return 0; } ++static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) ++ { return 0; } ++static inline void inc_slabs_node(struct kmem_cache *s, int node, ++ int objects) {} ++static inline void dec_slabs_node(struct kmem_cache *s, int node, ++ int objects) {} ++ ++#endif /* CONFIG_SLUB_DEBUG */ ++ ++/* ++ * Hooks for other subsystems that check memory allocations. In a typical ++ * production configuration these hooks all should produce no code at all. ++ */ ++static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) ++{ ++ kmemleak_alloc(ptr, size, 1, flags); ++ kasan_kmalloc_large(ptr, size); ++} ++ ++static inline void kfree_hook(const void *x) ++{ ++ kmemleak_free(x); ++ kasan_kfree_large(x); ++} ++ ++static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, ++ gfp_t flags) ++{ ++ flags &= gfp_allowed_mask; ++ lockdep_trace_alloc(flags); ++ might_sleep_if(flags & __GFP_WAIT); ++ ++ if (should_failslab(s->object_size, flags, s->flags)) ++ return NULL; ++ ++ return memcg_kmem_get_cache(s, flags); ++} ++ ++static inline void slab_post_alloc_hook(struct kmem_cache *s, ++ gfp_t flags, void *object) ++{ ++ flags &= gfp_allowed_mask; ++ kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); ++ kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); ++ memcg_kmem_put_cache(s); ++ kasan_slab_alloc(s, object); ++} ++ ++static inline void slab_free_hook(struct kmem_cache *s, void *x) ++{ ++ kmemleak_free_recursive(x, s->flags); ++ ++ /* ++ * Trouble is that we may no longer disable interrupts in the fast path ++ * So in order to make the debug calls that expect irqs to be ++ * disabled we need to disable interrupts temporarily. ++ */ ++#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP) ++ { ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ kmemcheck_slab_free(s, x, s->object_size); ++ debug_check_no_locks_freed(x, s->object_size); ++ local_irq_restore(flags); ++ } ++#endif ++ if (!(s->flags & SLAB_DEBUG_OBJECTS)) ++ debug_check_no_obj_freed(x, s->object_size); ++ ++ kasan_slab_free(s, x); ++} ++ ++/* ++ * Slab allocation and freeing ++ */ ++static inline struct page *alloc_slab_page(struct kmem_cache *s, ++ gfp_t flags, int node, struct kmem_cache_order_objects oo) ++{ ++ struct page *page; ++ int order = oo_order(oo); ++ ++ flags |= __GFP_NOTRACK; ++ ++ if (memcg_charge_slab(s, flags, order)) ++ return NULL; ++ ++ if (node == NUMA_NO_NODE) ++ page = alloc_pages(flags, order); ++ else ++ page = alloc_pages_exact_node(node, flags, order); ++ ++ if (!page) ++ memcg_uncharge_slab(s, order); ++ ++ return page; ++} ++ ++static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) ++{ ++ struct page *page; ++ struct kmem_cache_order_objects oo = s->oo; ++ gfp_t alloc_gfp; ++ ++ flags &= gfp_allowed_mask; ++ ++ if (flags & __GFP_WAIT) ++ local_irq_enable(); ++ ++ flags |= s->allocflags; ++ ++ /* ++ * Let the initial higher-order allocation fail under memory pressure ++ * so we fall-back to the minimum order allocation. ++ */ ++ alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; ++ ++ page = alloc_slab_page(s, alloc_gfp, node, oo); ++ if (unlikely(!page)) { ++ oo = s->min; ++ alloc_gfp = flags; ++ /* ++ * Allocation may have failed due to fragmentation. ++ * Try a lower order alloc if possible ++ */ ++ page = alloc_slab_page(s, alloc_gfp, node, oo); ++ ++ if (page) ++ stat(s, ORDER_FALLBACK); ++ } ++ ++ if (kmemcheck_enabled && page ++ && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { ++ int pages = 1 << oo_order(oo); ++ ++ kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node); ++ ++ /* ++ * Objects from caches that have a constructor don't get ++ * cleared when they're allocated, so we need to do it here. ++ */ ++ if (s->ctor) ++ kmemcheck_mark_uninitialized_pages(page, pages); ++ else ++ kmemcheck_mark_unallocated_pages(page, pages); ++ } ++ ++ if (flags & __GFP_WAIT) ++ local_irq_disable(); ++ if (!page) ++ return NULL; ++ ++ page->objects = oo_objects(oo); ++ mod_zone_page_state(page_zone(page), ++ (s->flags & SLAB_RECLAIM_ACCOUNT) ? ++ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, ++ 1 << oo_order(oo)); ++ ++ return page; ++} ++ ++static void setup_object(struct kmem_cache *s, struct page *page, ++ void *object) ++{ ++ setup_object_debug(s, page, object); ++ if (unlikely(s->ctor)) { ++ kasan_unpoison_object_data(s, object); ++ s->ctor(object); ++ kasan_poison_object_data(s, object); ++ } ++} ++ ++static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) ++{ ++ struct page *page; ++ void *start; ++ void *p; ++ int order; ++ int idx; ++ ++ if (unlikely(flags & GFP_SLAB_BUG_MASK)) { ++ pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); ++ BUG(); ++ } ++ ++ page = allocate_slab(s, ++ flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); ++ if (!page) ++ goto out; ++ ++ order = compound_order(page); ++ inc_slabs_node(s, page_to_nid(page), page->objects); ++ page->slab_cache = s; ++ __SetPageSlab(page); ++ if (page_is_pfmemalloc(page)) ++ SetPageSlabPfmemalloc(page); ++ ++ start = page_address(page); ++ ++ if (unlikely(s->flags & SLAB_POISON)) ++ memset(start, POISON_INUSE, PAGE_SIZE << order); ++ ++ kasan_poison_slab(page); ++ ++ for_each_object_idx(p, idx, s, start, page->objects) { ++ setup_object(s, page, p); ++ if (likely(idx < page->objects)) ++ set_freepointer(s, p, p + s->size); ++ else ++ set_freepointer(s, p, NULL); ++ } ++ ++ page->freelist = start; ++ page->inuse = page->objects; ++ page->frozen = 1; ++out: ++ return page; ++} ++ ++static void __free_slab(struct kmem_cache *s, struct page *page) ++{ ++ int order = compound_order(page); ++ int pages = 1 << order; ++ ++ if (kmem_cache_debug(s)) { ++ void *p; ++ ++ slab_pad_check(s, page); ++ for_each_object(p, s, page_address(page), ++ page->objects) ++ check_object(s, page, p, SLUB_RED_INACTIVE); ++ } ++ ++ kmemcheck_free_shadow(page, compound_order(page)); ++ ++ mod_zone_page_state(page_zone(page), ++ (s->flags & SLAB_RECLAIM_ACCOUNT) ? ++ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, ++ -pages); ++ ++ __ClearPageSlabPfmemalloc(page); ++ __ClearPageSlab(page); ++ ++ page_mapcount_reset(page); ++ if (current->reclaim_state) ++ current->reclaim_state->reclaimed_slab += pages; ++ __free_pages(page, order); ++ memcg_uncharge_slab(s, order); ++} ++ ++#define need_reserve_slab_rcu \ ++ (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head)) ++ ++static void rcu_free_slab(struct rcu_head *h) ++{ ++ struct page *page; ++ ++ if (need_reserve_slab_rcu) ++ page = virt_to_head_page(h); ++ else ++ page = container_of((struct list_head *)h, struct page, lru); ++ ++ __free_slab(page->slab_cache, page); ++} ++ ++static void free_slab(struct kmem_cache *s, struct page *page) ++{ ++ if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) { ++ struct rcu_head *head; ++ ++ if (need_reserve_slab_rcu) { ++ int order = compound_order(page); ++ int offset = (PAGE_SIZE << order) - s->reserved; ++ ++ VM_BUG_ON(s->reserved != sizeof(*head)); ++ head = page_address(page) + offset; ++ } else { ++ /* ++ * RCU free overloads the RCU head over the LRU ++ */ ++ head = (void *)&page->lru; ++ } ++ ++ call_rcu(head, rcu_free_slab); ++ } else ++ __free_slab(s, page); ++} ++ ++static void discard_slab(struct kmem_cache *s, struct page *page) ++{ ++ dec_slabs_node(s, page_to_nid(page), page->objects); ++ free_slab(s, page); ++} ++ ++/* ++ * Management of partially allocated slabs. ++ */ ++static inline void ++__add_partial(struct kmem_cache_node *n, struct page *page, int tail) ++{ ++ n->nr_partial++; ++ if (tail == DEACTIVATE_TO_TAIL) ++ list_add_tail(&page->lru, &n->partial); ++ else ++ list_add(&page->lru, &n->partial); ++} ++ ++static inline void add_partial(struct kmem_cache_node *n, ++ struct page *page, int tail) ++{ ++ lockdep_assert_held(&n->list_lock); ++ __add_partial(n, page, tail); ++} ++ ++static inline void ++__remove_partial(struct kmem_cache_node *n, struct page *page) ++{ ++ list_del(&page->lru); ++ n->nr_partial--; ++} ++ ++static inline void remove_partial(struct kmem_cache_node *n, ++ struct page *page) ++{ ++ lockdep_assert_held(&n->list_lock); ++ __remove_partial(n, page); ++} ++ ++/* ++ * Remove slab from the partial list, freeze it and ++ * return the pointer to the freelist. ++ * ++ * Returns a list of objects or NULL if it fails. ++ */ ++static inline void *acquire_slab(struct kmem_cache *s, ++ struct kmem_cache_node *n, struct page *page, ++ int mode, int *objects) ++{ ++ void *freelist; ++ unsigned long counters; ++ struct page new; ++ ++ lockdep_assert_held(&n->list_lock); ++ ++ /* ++ * Zap the freelist and set the frozen bit. ++ * The old freelist is the list of objects for the ++ * per cpu allocation list. ++ */ ++ freelist = page->freelist; ++ counters = page->counters; ++ new.counters = counters; ++ *objects = new.objects - new.inuse; ++ if (mode) { ++ new.inuse = page->objects; ++ new.freelist = NULL; ++ } else { ++ new.freelist = freelist; ++ } ++ ++ VM_BUG_ON(new.frozen); ++ new.frozen = 1; ++ ++ if (!__cmpxchg_double_slab(s, page, ++ freelist, counters, ++ new.freelist, new.counters, ++ "acquire_slab")) ++ return NULL; ++ ++ remove_partial(n, page); ++ WARN_ON(!freelist); ++ return freelist; ++} ++ ++static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain); ++static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags); ++ ++/* ++ * Try to allocate a partial slab from a specific node. ++ */ ++static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, ++ struct kmem_cache_cpu *c, gfp_t flags) ++{ ++ struct page *page, *page2; ++ void *object = NULL; ++ int available = 0; ++ int objects; ++ ++ /* ++ * Racy check. If we mistakenly see no partial slabs then we ++ * just allocate an empty slab. If we mistakenly try to get a ++ * partial slab and there is none available then get_partials() ++ * will return NULL. ++ */ ++ if (!n || !n->nr_partial) ++ return NULL; ++ ++ spin_lock(&n->list_lock); ++ list_for_each_entry_safe(page, page2, &n->partial, lru) { ++ void *t; ++ ++ if (!pfmemalloc_match(page, flags)) ++ continue; ++ ++ t = acquire_slab(s, n, page, object == NULL, &objects); ++ if (!t) ++ break; ++ ++ available += objects; ++ if (!object) { ++ c->page = page; ++ stat(s, ALLOC_FROM_PARTIAL); ++ object = t; ++ } else { ++ put_cpu_partial(s, page, 0); ++ stat(s, CPU_PARTIAL_NODE); ++ } ++ if (!kmem_cache_has_cpu_partial(s) ++ || available > s->cpu_partial / 2) ++ break; ++ ++ } ++ spin_unlock(&n->list_lock); ++ return object; ++} ++ ++/* ++ * Get a page from somewhere. Search in increasing NUMA distances. ++ */ ++static void *get_any_partial(struct kmem_cache *s, gfp_t flags, ++ struct kmem_cache_cpu *c) ++{ ++#ifdef CONFIG_NUMA ++ struct zonelist *zonelist; ++ struct zoneref *z; ++ struct zone *zone; ++ enum zone_type high_zoneidx = gfp_zone(flags); ++ void *object; ++ unsigned int cpuset_mems_cookie; ++ ++ /* ++ * The defrag ratio allows a configuration of the tradeoffs between ++ * inter node defragmentation and node local allocations. A lower ++ * defrag_ratio increases the tendency to do local allocations ++ * instead of attempting to obtain partial slabs from other nodes. ++ * ++ * If the defrag_ratio is set to 0 then kmalloc() always ++ * returns node local objects. If the ratio is higher then kmalloc() ++ * may return off node objects because partial slabs are obtained ++ * from other nodes and filled up. ++ * ++ * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes ++ * defrag_ratio = 1000) then every (well almost) allocation will ++ * first attempt to defrag slab caches on other nodes. This means ++ * scanning over all nodes to look for partial slabs which may be ++ * expensive if we do it every time we are trying to find a slab ++ * with available objects. ++ */ ++ if (!s->remote_node_defrag_ratio || ++ get_cycles() % 1024 > s->remote_node_defrag_ratio) ++ return NULL; ++ ++ do { ++ cpuset_mems_cookie = read_mems_allowed_begin(); ++ zonelist = node_zonelist(mempolicy_slab_node(), flags); ++ for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { ++ struct kmem_cache_node *n; ++ ++ n = get_node(s, zone_to_nid(zone)); ++ ++ if (n && cpuset_zone_allowed(zone, flags) && ++ n->nr_partial > s->min_partial) { ++ object = get_partial_node(s, n, c, flags); ++ if (object) { ++ /* ++ * Don't check read_mems_allowed_retry() ++ * here - if mems_allowed was updated in ++ * parallel, that was a harmless race ++ * between allocation and the cpuset ++ * update ++ */ ++ return object; ++ } ++ } ++ } ++ } while (read_mems_allowed_retry(cpuset_mems_cookie)); ++#endif ++ return NULL; ++} ++ ++/* ++ * Get a partial page, lock it and return it. ++ */ ++static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, ++ struct kmem_cache_cpu *c) ++{ ++ void *object; ++ int searchnode = node; ++ ++ if (node == NUMA_NO_NODE) ++ searchnode = numa_mem_id(); ++ else if (!node_present_pages(node)) ++ searchnode = node_to_mem_node(node); ++ ++ object = get_partial_node(s, get_node(s, searchnode), c, flags); ++ if (object || node != NUMA_NO_NODE) ++ return object; ++ ++ return get_any_partial(s, flags, c); ++} ++ ++#ifdef CONFIG_PREEMPT ++/* ++ * Calculate the next globally unique transaction for disambiguiation ++ * during cmpxchg. The transactions start with the cpu number and are then ++ * incremented by CONFIG_NR_CPUS. ++ */ ++#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS) ++#else ++/* ++ * No preemption supported therefore also no need to check for ++ * different cpus. ++ */ ++#define TID_STEP 1 ++#endif ++ ++static inline unsigned long next_tid(unsigned long tid) ++{ ++ return tid + TID_STEP; ++} ++ ++static inline unsigned int tid_to_cpu(unsigned long tid) ++{ ++ return tid % TID_STEP; ++} ++ ++static inline unsigned long tid_to_event(unsigned long tid) ++{ ++ return tid / TID_STEP; ++} ++ ++static inline unsigned int init_tid(int cpu) ++{ ++ return cpu; ++} ++ ++static inline void note_cmpxchg_failure(const char *n, ++ const struct kmem_cache *s, unsigned long tid) ++{ ++#ifdef SLUB_DEBUG_CMPXCHG ++ unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); ++ ++ pr_info("%s %s: cmpxchg redo ", n, s->name); ++ ++#ifdef CONFIG_PREEMPT ++ if (tid_to_cpu(tid) != tid_to_cpu(actual_tid)) ++ pr_warn("due to cpu change %d -> %d\n", ++ tid_to_cpu(tid), tid_to_cpu(actual_tid)); ++ else ++#endif ++ if (tid_to_event(tid) != tid_to_event(actual_tid)) ++ pr_warn("due to cpu running other code. Event %ld->%ld\n", ++ tid_to_event(tid), tid_to_event(actual_tid)); ++ else ++ pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n", ++ actual_tid, tid, next_tid(tid)); ++#endif ++ stat(s, CMPXCHG_DOUBLE_CPU_FAIL); ++} ++ ++static void init_kmem_cache_cpus(struct kmem_cache *s) ++{ ++ int cpu; ++ ++ for_each_possible_cpu(cpu) ++ per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); ++} ++ ++/* ++ * Remove the cpu slab ++ */ ++static void deactivate_slab(struct kmem_cache *s, struct page *page, ++ void *freelist) ++{ ++ enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; ++ struct kmem_cache_node *n = get_node(s, page_to_nid(page)); ++ int lock = 0; ++ enum slab_modes l = M_NONE, m = M_NONE; ++ void *nextfree; ++ int tail = DEACTIVATE_TO_HEAD; ++ struct page new; ++ struct page old; ++ ++ if (page->freelist) { ++ stat(s, DEACTIVATE_REMOTE_FREES); ++ tail = DEACTIVATE_TO_TAIL; ++ } ++ ++ /* ++ * Stage one: Free all available per cpu objects back ++ * to the page freelist while it is still frozen. Leave the ++ * last one. ++ * ++ * There is no need to take the list->lock because the page ++ * is still frozen. ++ */ ++ while (freelist && (nextfree = get_freepointer(s, freelist))) { ++ void *prior; ++ unsigned long counters; ++ ++ do { ++ prior = page->freelist; ++ counters = page->counters; ++ set_freepointer(s, freelist, prior); ++ new.counters = counters; ++ new.inuse--; ++ VM_BUG_ON(!new.frozen); ++ ++ } while (!__cmpxchg_double_slab(s, page, ++ prior, counters, ++ freelist, new.counters, ++ "drain percpu freelist")); ++ ++ freelist = nextfree; ++ } ++ ++ /* ++ * Stage two: Ensure that the page is unfrozen while the ++ * list presence reflects the actual number of objects ++ * during unfreeze. ++ * ++ * We setup the list membership and then perform a cmpxchg ++ * with the count. If there is a mismatch then the page ++ * is not unfrozen but the page is on the wrong list. ++ * ++ * Then we restart the process which may have to remove ++ * the page from the list that we just put it on again ++ * because the number of objects in the slab may have ++ * changed. ++ */ ++redo: ++ ++ old.freelist = page->freelist; ++ old.counters = page->counters; ++ VM_BUG_ON(!old.frozen); ++ ++ /* Determine target state of the slab */ ++ new.counters = old.counters; ++ if (freelist) { ++ new.inuse--; ++ set_freepointer(s, freelist, old.freelist); ++ new.freelist = freelist; ++ } else ++ new.freelist = old.freelist; ++ ++ new.frozen = 0; ++ ++ if (!new.inuse && n->nr_partial >= s->min_partial) ++ m = M_FREE; ++ else if (new.freelist) { ++ m = M_PARTIAL; ++ if (!lock) { ++ lock = 1; ++ /* ++ * Taking the spinlock removes the possiblity ++ * that acquire_slab() will see a slab page that ++ * is frozen ++ */ ++ spin_lock(&n->list_lock); ++ } ++ } else { ++ m = M_FULL; ++ if (kmem_cache_debug(s) && !lock) { ++ lock = 1; ++ /* ++ * This also ensures that the scanning of full ++ * slabs from diagnostic functions will not see ++ * any frozen slabs. ++ */ ++ spin_lock(&n->list_lock); ++ } ++ } ++ ++ if (l != m) { ++ ++ if (l == M_PARTIAL) ++ ++ remove_partial(n, page); ++ ++ else if (l == M_FULL) ++ ++ remove_full(s, n, page); ++ ++ if (m == M_PARTIAL) { ++ ++ add_partial(n, page, tail); ++ stat(s, tail); ++ ++ } else if (m == M_FULL) { ++ ++ stat(s, DEACTIVATE_FULL); ++ add_full(s, n, page); ++ ++ } ++ } ++ ++ l = m; ++ if (!__cmpxchg_double_slab(s, page, ++ old.freelist, old.counters, ++ new.freelist, new.counters, ++ "unfreezing slab")) ++ goto redo; ++ ++ if (lock) ++ spin_unlock(&n->list_lock); ++ ++ if (m == M_FREE) { ++ stat(s, DEACTIVATE_EMPTY); ++ discard_slab(s, page); ++ stat(s, FREE_SLAB); ++ } ++} ++ ++/* ++ * Unfreeze all the cpu partial slabs. ++ * ++ * This function must be called with interrupts disabled ++ * for the cpu using c (or some other guarantee must be there ++ * to guarantee no concurrent accesses). ++ */ ++static void unfreeze_partials(struct kmem_cache *s, ++ struct kmem_cache_cpu *c) ++{ ++#ifdef CONFIG_SLUB_CPU_PARTIAL ++ struct kmem_cache_node *n = NULL, *n2 = NULL; ++ struct page *page, *discard_page = NULL; ++ ++ while ((page = c->partial)) { ++ struct page new; ++ struct page old; ++ ++ c->partial = page->next; ++ ++ n2 = get_node(s, page_to_nid(page)); ++ if (n != n2) { ++ if (n) ++ spin_unlock(&n->list_lock); ++ ++ n = n2; ++ spin_lock(&n->list_lock); ++ } ++ ++ do { ++ ++ old.freelist = page->freelist; ++ old.counters = page->counters; ++ VM_BUG_ON(!old.frozen); ++ ++ new.counters = old.counters; ++ new.freelist = old.freelist; ++ ++ new.frozen = 0; ++ ++ } while (!__cmpxchg_double_slab(s, page, ++ old.freelist, old.counters, ++ new.freelist, new.counters, ++ "unfreezing slab")); ++ ++ if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) { ++ page->next = discard_page; ++ discard_page = page; ++ } else { ++ add_partial(n, page, DEACTIVATE_TO_TAIL); ++ stat(s, FREE_ADD_PARTIAL); ++ } ++ } ++ ++ if (n) ++ spin_unlock(&n->list_lock); ++ ++ while (discard_page) { ++ page = discard_page; ++ discard_page = discard_page->next; ++ ++ stat(s, DEACTIVATE_EMPTY); ++ discard_slab(s, page); ++ stat(s, FREE_SLAB); ++ } ++#endif ++} ++ ++/* ++ * Put a page that was just frozen (in __slab_free) into a partial page ++ * slot if available. This is done without interrupts disabled and without ++ * preemption disabled. The cmpxchg is racy and may put the partial page ++ * onto a random cpus partial slot. ++ * ++ * If we did not find a slot then simply move all the partials to the ++ * per node partial list. ++ */ ++static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) ++{ ++#ifdef CONFIG_SLUB_CPU_PARTIAL ++ struct page *oldpage; ++ int pages; ++ int pobjects; ++ ++ preempt_disable(); ++ do { ++ pages = 0; ++ pobjects = 0; ++ oldpage = this_cpu_read(s->cpu_slab->partial); ++ ++ if (oldpage) { ++ pobjects = oldpage->pobjects; ++ pages = oldpage->pages; ++ if (drain && pobjects > s->cpu_partial) { ++ unsigned long flags; ++ /* ++ * partial array is full. Move the existing ++ * set to the per node partial list. ++ */ ++ local_irq_save(flags); ++ unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); ++ local_irq_restore(flags); ++ oldpage = NULL; ++ pobjects = 0; ++ pages = 0; ++ stat(s, CPU_PARTIAL_DRAIN); ++ } ++ } ++ ++ pages++; ++ pobjects += page->objects - page->inuse; ++ ++ page->pages = pages; ++ page->pobjects = pobjects; ++ page->next = oldpage; ++ ++ } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) ++ != oldpage); ++ if (unlikely(!s->cpu_partial)) { ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); ++ local_irq_restore(flags); ++ } ++ preempt_enable(); ++#endif ++} ++ ++static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) ++{ ++ stat(s, CPUSLAB_FLUSH); ++ deactivate_slab(s, c->page, c->freelist); ++ ++ c->tid = next_tid(c->tid); ++ c->page = NULL; ++ c->freelist = NULL; ++} ++ ++/* ++ * Flush cpu slab. ++ * ++ * Called from IPI handler with interrupts disabled. ++ */ ++static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) ++{ ++ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); ++ ++ if (likely(c)) { ++ if (c->page) ++ flush_slab(s, c); ++ ++ unfreeze_partials(s, c); ++ } ++} ++ ++static void flush_cpu_slab(void *d) ++{ ++ struct kmem_cache *s = d; ++ ++ __flush_cpu_slab(s, smp_processor_id()); ++} ++ ++static bool has_cpu_slab(int cpu, void *info) ++{ ++ struct kmem_cache *s = info; ++ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); ++ ++ return c->page || c->partial; ++} ++ ++static void flush_all(struct kmem_cache *s) ++{ ++ on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); ++} ++ ++/* ++ * Check if the objects in a per cpu structure fit numa ++ * locality expectations. ++ */ ++static inline int node_match(struct page *page, int node) ++{ ++#ifdef CONFIG_NUMA ++ if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node)) ++ return 0; ++#endif ++ return 1; ++} ++ ++#ifdef CONFIG_SLUB_DEBUG ++static int count_free(struct page *page) ++{ ++ return page->objects - page->inuse; ++} ++ ++static inline unsigned long node_nr_objs(struct kmem_cache_node *n) ++{ ++ return atomic_long_read(&n->total_objects); ++} ++#endif /* CONFIG_SLUB_DEBUG */ ++ ++#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS) ++static unsigned long count_partial(struct kmem_cache_node *n, ++ int (*get_count)(struct page *)) ++{ ++ unsigned long flags; ++ unsigned long x = 0; ++ struct page *page; ++ ++ spin_lock_irqsave(&n->list_lock, flags); ++ list_for_each_entry(page, &n->partial, lru) ++ x += get_count(page); ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ return x; ++} ++#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */ ++ ++static noinline void ++slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) ++{ ++#ifdef CONFIG_SLUB_DEBUG ++ static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, ++ DEFAULT_RATELIMIT_BURST); ++ int node; ++ struct kmem_cache_node *n; ++ ++ if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) ++ return; ++ ++ pr_warn("SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n", ++ nid, gfpflags); ++ pr_warn(" cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n", ++ s->name, s->object_size, s->size, oo_order(s->oo), ++ oo_order(s->min)); ++ ++ if (oo_order(s->min) > get_order(s->object_size)) ++ pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n", ++ s->name); ++ ++ for_each_kmem_cache_node(s, node, n) { ++ unsigned long nr_slabs; ++ unsigned long nr_objs; ++ unsigned long nr_free; ++ ++ nr_free = count_partial(n, count_free); ++ nr_slabs = node_nr_slabs(n); ++ nr_objs = node_nr_objs(n); ++ ++ pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n", ++ node, nr_slabs, nr_objs, nr_free); ++ } ++#endif ++} ++ ++static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, ++ int node, struct kmem_cache_cpu **pc) ++{ ++ void *freelist; ++ struct kmem_cache_cpu *c = *pc; ++ struct page *page; ++ ++ freelist = get_partial(s, flags, node, c); ++ ++ if (freelist) ++ return freelist; ++ ++ page = new_slab(s, flags, node); ++ if (page) { ++ c = raw_cpu_ptr(s->cpu_slab); ++ if (c->page) ++ flush_slab(s, c); ++ ++ /* ++ * No other reference to the page yet so we can ++ * muck around with it freely without cmpxchg ++ */ ++ freelist = page->freelist; ++ page->freelist = NULL; ++ ++ stat(s, ALLOC_SLAB); ++ c->page = page; ++ *pc = c; ++ } else ++ freelist = NULL; ++ ++ return freelist; ++} ++ ++static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags) ++{ ++ if (unlikely(PageSlabPfmemalloc(page))) ++ return gfp_pfmemalloc_allowed(gfpflags); ++ ++ return true; ++} ++ ++/* ++ * Check the page->freelist of a page and either transfer the freelist to the ++ * per cpu freelist or deactivate the page. ++ * ++ * The page is still frozen if the return value is not NULL. ++ * ++ * If this function returns NULL then the page has been unfrozen. ++ * ++ * This function must be called with interrupt disabled. ++ */ ++static inline void *get_freelist(struct kmem_cache *s, struct page *page) ++{ ++ struct page new; ++ unsigned long counters; ++ void *freelist; ++ ++ do { ++ freelist = page->freelist; ++ counters = page->counters; ++ ++ new.counters = counters; ++ VM_BUG_ON(!new.frozen); ++ ++ new.inuse = page->objects; ++ new.frozen = freelist != NULL; ++ ++ } while (!__cmpxchg_double_slab(s, page, ++ freelist, counters, ++ NULL, new.counters, ++ "get_freelist")); ++ ++ return freelist; ++} ++ ++/* ++ * Slow path. The lockless freelist is empty or we need to perform ++ * debugging duties. ++ * ++ * Processing is still very fast if new objects have been freed to the ++ * regular freelist. In that case we simply take over the regular freelist ++ * as the lockless freelist and zap the regular freelist. ++ * ++ * If that is not working then we fall back to the partial lists. We take the ++ * first element of the freelist as the object to allocate now and move the ++ * rest of the freelist to the lockless freelist. ++ * ++ * And if we were unable to get a new slab from the partial slab lists then ++ * we need to allocate a new slab. This is the slowest path since it involves ++ * a call to the page allocator and the setup of a new slab. ++ */ ++static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ++ unsigned long addr, struct kmem_cache_cpu *c) ++{ ++ void *freelist; ++ struct page *page; ++ unsigned long flags; ++ ++ local_irq_save(flags); ++#ifdef CONFIG_PREEMPT ++ /* ++ * We may have been preempted and rescheduled on a different ++ * cpu before disabling interrupts. Need to reload cpu area ++ * pointer. ++ */ ++ c = this_cpu_ptr(s->cpu_slab); ++#endif ++ ++ page = c->page; ++ if (!page) ++ goto new_slab; ++redo: ++ ++ if (unlikely(!node_match(page, node))) { ++ int searchnode = node; ++ ++ if (node != NUMA_NO_NODE && !node_present_pages(node)) ++ searchnode = node_to_mem_node(node); ++ ++ if (unlikely(!node_match(page, searchnode))) { ++ stat(s, ALLOC_NODE_MISMATCH); ++ deactivate_slab(s, page, c->freelist); ++ c->page = NULL; ++ c->freelist = NULL; ++ goto new_slab; ++ } ++ } ++ ++ /* ++ * By rights, we should be searching for a slab page that was ++ * PFMEMALLOC but right now, we are losing the pfmemalloc ++ * information when the page leaves the per-cpu allocator ++ */ ++ if (unlikely(!pfmemalloc_match(page, gfpflags))) { ++ deactivate_slab(s, page, c->freelist); ++ c->page = NULL; ++ c->freelist = NULL; ++ goto new_slab; ++ } ++ ++ /* must check again c->freelist in case of cpu migration or IRQ */ ++ freelist = c->freelist; ++ if (freelist) ++ goto load_freelist; ++ ++ freelist = get_freelist(s, page); ++ ++ if (!freelist) { ++ c->page = NULL; ++ stat(s, DEACTIVATE_BYPASS); ++ goto new_slab; ++ } ++ ++ stat(s, ALLOC_REFILL); ++ ++load_freelist: ++ /* ++ * freelist is pointing to the list of objects to be used. ++ * page is pointing to the page from which the objects are obtained. ++ * That page must be frozen for per cpu allocations to work. ++ */ ++ VM_BUG_ON(!c->page->frozen); ++ c->freelist = get_freepointer(s, freelist); ++ c->tid = next_tid(c->tid); ++ local_irq_restore(flags); ++ return freelist; ++ ++new_slab: ++ ++ if (c->partial) { ++ page = c->page = c->partial; ++ c->partial = page->next; ++ stat(s, CPU_PARTIAL_ALLOC); ++ c->freelist = NULL; ++ goto redo; ++ } ++ ++ freelist = new_slab_objects(s, gfpflags, node, &c); ++ ++ if (unlikely(!freelist)) { ++ slab_out_of_memory(s, gfpflags, node); ++ local_irq_restore(flags); ++ return NULL; ++ } ++ ++ page = c->page; ++ if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) ++ goto load_freelist; ++ ++ /* Only entered in the debug case */ ++ if (kmem_cache_debug(s) && ++ !alloc_debug_processing(s, page, freelist, addr)) ++ goto new_slab; /* Slab failed checks. Next slab needed */ ++ ++ deactivate_slab(s, page, get_freepointer(s, freelist)); ++ c->page = NULL; ++ c->freelist = NULL; ++ local_irq_restore(flags); ++ return freelist; ++} ++ ++/* ++ * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) ++ * have the fastpath folded into their functions. So no function call ++ * overhead for requests that can be satisfied on the fastpath. ++ * ++ * The fastpath works by first checking if the lockless freelist can be used. ++ * If not then __slab_alloc is called for slow processing. ++ * ++ * Otherwise we can simply pick the next object from the lockless free list. ++ */ ++static __always_inline void *slab_alloc_node(struct kmem_cache *s, ++ gfp_t gfpflags, int node, unsigned long addr) ++{ ++ void **object; ++ struct kmem_cache_cpu *c; ++ struct page *page; ++ unsigned long tid; ++ ++ s = slab_pre_alloc_hook(s, gfpflags); ++ if (!s) ++ return NULL; ++redo: ++ /* ++ * Must read kmem_cache cpu data via this cpu ptr. Preemption is ++ * enabled. We may switch back and forth between cpus while ++ * reading from one cpu area. That does not matter as long ++ * as we end up on the original cpu again when doing the cmpxchg. ++ * ++ * We should guarantee that tid and kmem_cache are retrieved on ++ * the same cpu. It could be different if CONFIG_PREEMPT so we need ++ * to check if it is matched or not. ++ */ ++ do { ++ tid = this_cpu_read(s->cpu_slab->tid); ++ c = raw_cpu_ptr(s->cpu_slab); ++ } while (IS_ENABLED(CONFIG_PREEMPT) && ++ unlikely(tid != READ_ONCE(c->tid))); ++ ++ /* ++ * Irqless object alloc/free algorithm used here depends on sequence ++ * of fetching cpu_slab's data. tid should be fetched before anything ++ * on c to guarantee that object and page associated with previous tid ++ * won't be used with current tid. If we fetch tid first, object and ++ * page could be one associated with next tid and our alloc/free ++ * request will be failed. In this case, we will retry. So, no problem. ++ */ ++ barrier(); ++ ++ /* ++ * The transaction ids are globally unique per cpu and per operation on ++ * a per cpu queue. Thus they can be guarantee that the cmpxchg_double ++ * occurs on the right processor and that there was no operation on the ++ * linked list in between. ++ */ ++ ++ object = c->freelist; ++ page = c->page; ++ if (unlikely(!object || !node_match(page, node))) { ++ object = __slab_alloc(s, gfpflags, node, addr, c); ++ stat(s, ALLOC_SLOWPATH); ++ } else { ++ void *next_object = get_freepointer_safe(s, object); ++ ++ /* ++ * The cmpxchg will only match if there was no additional ++ * operation and if we are on the right processor. ++ * ++ * The cmpxchg does the following atomically (without lock ++ * semantics!) ++ * 1. Relocate first pointer to the current per cpu area. ++ * 2. Verify that tid and freelist have not been changed ++ * 3. If they were not changed replace tid and freelist ++ * ++ * Since this is without lock semantics the protection is only ++ * against code executing on this cpu *not* from access by ++ * other cpus. ++ */ ++ if (unlikely(!this_cpu_cmpxchg_double( ++ s->cpu_slab->freelist, s->cpu_slab->tid, ++ object, tid, ++ next_object, next_tid(tid)))) { ++ ++ note_cmpxchg_failure("slab_alloc", s, tid); ++ goto redo; ++ } ++ prefetch_freepointer(s, next_object); ++ stat(s, ALLOC_FASTPATH); ++ } ++ ++ if (unlikely(gfpflags & __GFP_ZERO) && object) ++ memset(object, 0, s->object_size); ++ ++ slab_post_alloc_hook(s, gfpflags, object); ++ ++ return object; ++} ++ ++static __always_inline void *slab_alloc(struct kmem_cache *s, ++ gfp_t gfpflags, unsigned long addr) ++{ ++ return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr); ++} ++ ++void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) ++{ ++ void *ret = slab_alloc(s, gfpflags, _RET_IP_); ++ ++ trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, ++ s->size, gfpflags); ++ ++ return ret; ++} ++EXPORT_SYMBOL(kmem_cache_alloc); ++ ++#ifdef CONFIG_TRACING ++void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) ++{ ++ void *ret = slab_alloc(s, gfpflags, _RET_IP_); ++ trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); ++ kasan_kmalloc(s, ret, size); ++ return ret; ++} ++EXPORT_SYMBOL(kmem_cache_alloc_trace); ++#endif ++ ++#ifdef CONFIG_NUMA ++void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) ++{ ++ void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); ++ ++ trace_kmem_cache_alloc_node(_RET_IP_, ret, ++ s->object_size, s->size, gfpflags, node); ++ ++ return ret; ++} ++EXPORT_SYMBOL(kmem_cache_alloc_node); ++ ++#ifdef CONFIG_TRACING ++void *kmem_cache_alloc_node_trace(struct kmem_cache *s, ++ gfp_t gfpflags, ++ int node, size_t size) ++{ ++ void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); ++ ++ trace_kmalloc_node(_RET_IP_, ret, ++ size, s->size, gfpflags, node); ++ ++ kasan_kmalloc(s, ret, size); ++ return ret; ++} ++EXPORT_SYMBOL(kmem_cache_alloc_node_trace); ++#endif ++#endif ++ ++/* ++ * Slow path handling. This may still be called frequently since objects ++ * have a longer lifetime than the cpu slabs in most processing loads. ++ * ++ * So we still attempt to reduce cache line usage. Just take the slab ++ * lock and free the item. If there is no additional partial page ++ * handling required then we can return immediately. ++ */ ++static void __slab_free(struct kmem_cache *s, struct page *page, ++ void *x, unsigned long addr) ++{ ++ void *prior; ++ void **object = (void *)x; ++ int was_frozen; ++ struct page new; ++ unsigned long counters; ++ struct kmem_cache_node *n = NULL; ++ unsigned long uninitialized_var(flags); ++ ++ stat(s, FREE_SLOWPATH); ++ ++ if (kmem_cache_debug(s) && ++ !(n = free_debug_processing(s, page, x, addr, &flags))) ++ return; ++ ++ do { ++ if (unlikely(n)) { ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ n = NULL; ++ } ++ prior = page->freelist; ++ counters = page->counters; ++ set_freepointer(s, object, prior); ++ new.counters = counters; ++ was_frozen = new.frozen; ++ new.inuse--; ++ if ((!new.inuse || !prior) && !was_frozen) { ++ ++ if (kmem_cache_has_cpu_partial(s) && !prior) { ++ ++ /* ++ * Slab was on no list before and will be ++ * partially empty ++ * We can defer the list move and instead ++ * freeze it. ++ */ ++ new.frozen = 1; ++ ++ } else { /* Needs to be taken off a list */ ++ ++ n = get_node(s, page_to_nid(page)); ++ /* ++ * Speculatively acquire the list_lock. ++ * If the cmpxchg does not succeed then we may ++ * drop the list_lock without any processing. ++ * ++ * Otherwise the list_lock will synchronize with ++ * other processors updating the list of slabs. ++ */ ++ spin_lock_irqsave(&n->list_lock, flags); ++ ++ } ++ } ++ ++ } while (!cmpxchg_double_slab(s, page, ++ prior, counters, ++ object, new.counters, ++ "__slab_free")); ++ ++ if (likely(!n)) { ++ ++ /* ++ * If we just froze the page then put it onto the ++ * per cpu partial list. ++ */ ++ if (new.frozen && !was_frozen) { ++ put_cpu_partial(s, page, 1); ++ stat(s, CPU_PARTIAL_FREE); ++ } ++ /* ++ * The list lock was not taken therefore no list ++ * activity can be necessary. ++ */ ++ if (was_frozen) ++ stat(s, FREE_FROZEN); ++ return; ++ } ++ ++ if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) ++ goto slab_empty; ++ ++ /* ++ * Objects left in the slab. If it was not on the partial list before ++ * then add it. ++ */ ++ if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { ++ if (kmem_cache_debug(s)) ++ remove_full(s, n, page); ++ add_partial(n, page, DEACTIVATE_TO_TAIL); ++ stat(s, FREE_ADD_PARTIAL); ++ } ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ return; ++ ++slab_empty: ++ if (prior) { ++ /* ++ * Slab on the partial list. ++ */ ++ remove_partial(n, page); ++ stat(s, FREE_REMOVE_PARTIAL); ++ } else { ++ /* Slab must be on the full list */ ++ remove_full(s, n, page); ++ } ++ ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ stat(s, FREE_SLAB); ++ discard_slab(s, page); ++} ++ ++/* ++ * Fastpath with forced inlining to produce a kfree and kmem_cache_free that ++ * can perform fastpath freeing without additional function calls. ++ * ++ * The fastpath is only possible if we are freeing to the current cpu slab ++ * of this processor. This typically the case if we have just allocated ++ * the item before. ++ * ++ * If fastpath is not possible then fall back to __slab_free where we deal ++ * with all sorts of special processing. ++ */ ++static __always_inline void slab_free(struct kmem_cache *s, ++ struct page *page, void *x, unsigned long addr) ++{ ++ void **object = (void *)x; ++ struct kmem_cache_cpu *c; ++ unsigned long tid; ++ ++ slab_free_hook(s, x); ++ ++redo: ++ /* ++ * Determine the currently cpus per cpu slab. ++ * The cpu may change afterward. However that does not matter since ++ * data is retrieved via this pointer. If we are on the same cpu ++ * during the cmpxchg then the free will succedd. ++ */ ++ do { ++ tid = this_cpu_read(s->cpu_slab->tid); ++ c = raw_cpu_ptr(s->cpu_slab); ++ } while (IS_ENABLED(CONFIG_PREEMPT) && ++ unlikely(tid != READ_ONCE(c->tid))); ++ ++ /* Same with comment on barrier() in slab_alloc_node() */ ++ barrier(); ++ ++ if (likely(page == c->page)) { ++ set_freepointer(s, object, c->freelist); ++ ++ if (unlikely(!this_cpu_cmpxchg_double( ++ s->cpu_slab->freelist, s->cpu_slab->tid, ++ c->freelist, tid, ++ object, next_tid(tid)))) { ++ ++ note_cmpxchg_failure("slab_free", s, tid); ++ goto redo; ++ } ++ stat(s, FREE_FASTPATH); ++ } else ++ __slab_free(s, page, x, addr); ++ ++} ++ ++void kmem_cache_free(struct kmem_cache *s, void *x) ++{ ++ s = cache_from_obj(s, x); ++ if (!s) ++ return; ++ slab_free(s, virt_to_head_page(x), x, _RET_IP_); ++ trace_kmem_cache_free(_RET_IP_, x); ++} ++EXPORT_SYMBOL(kmem_cache_free); ++ ++/* ++ * Object placement in a slab is made very easy because we always start at ++ * offset 0. If we tune the size of the object to the alignment then we can ++ * get the required alignment by putting one properly sized object after ++ * another. ++ * ++ * Notice that the allocation order determines the sizes of the per cpu ++ * caches. Each processor has always one slab available for allocations. ++ * Increasing the allocation order reduces the number of times that slabs ++ * must be moved on and off the partial lists and is therefore a factor in ++ * locking overhead. ++ */ ++ ++/* ++ * Mininum / Maximum order of slab pages. This influences locking overhead ++ * and slab fragmentation. A higher order reduces the number of partial slabs ++ * and increases the number of allocations possible without having to ++ * take the list_lock. ++ */ ++static int slub_min_order; ++static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; ++static int slub_min_objects; ++ ++/* ++ * Calculate the order of allocation given an slab object size. ++ * ++ * The order of allocation has significant impact on performance and other ++ * system components. Generally order 0 allocations should be preferred since ++ * order 0 does not cause fragmentation in the page allocator. Larger objects ++ * be problematic to put into order 0 slabs because there may be too much ++ * unused space left. We go to a higher order if more than 1/16th of the slab ++ * would be wasted. ++ * ++ * In order to reach satisfactory performance we must ensure that a minimum ++ * number of objects is in one slab. Otherwise we may generate too much ++ * activity on the partial lists which requires taking the list_lock. This is ++ * less a concern for large slabs though which are rarely used. ++ * ++ * slub_max_order specifies the order where we begin to stop considering the ++ * number of objects in a slab as critical. If we reach slub_max_order then ++ * we try to keep the page order as low as possible. So we accept more waste ++ * of space in favor of a small page order. ++ * ++ * Higher order allocations also allow the placement of more objects in a ++ * slab and thereby reduce object handling overhead. If the user has ++ * requested a higher mininum order then we start with that one instead of ++ * the smallest order which will fit the object. ++ */ ++static inline int slab_order(int size, int min_objects, ++ int max_order, int fract_leftover, int reserved) ++{ ++ int order; ++ int rem; ++ int min_order = slub_min_order; ++ ++ if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE) ++ return get_order(size * MAX_OBJS_PER_PAGE) - 1; ++ ++ for (order = max(min_order, ++ fls(min_objects * size - 1) - PAGE_SHIFT); ++ order <= max_order; order++) { ++ ++ unsigned long slab_size = PAGE_SIZE << order; ++ ++ if (slab_size < min_objects * size + reserved) ++ continue; ++ ++ rem = (slab_size - reserved) % size; ++ ++ if (rem <= slab_size / fract_leftover) ++ break; ++ ++ } ++ ++ return order; ++} ++ ++static inline int calculate_order(int size, int reserved) ++{ ++ int order; ++ int min_objects; ++ int fraction; ++ int max_objects; ++ ++ /* ++ * Attempt to find best configuration for a slab. This ++ * works by first attempting to generate a layout with ++ * the best configuration and backing off gradually. ++ * ++ * First we reduce the acceptable waste in a slab. Then ++ * we reduce the minimum objects required in a slab. ++ */ ++ min_objects = slub_min_objects; ++ if (!min_objects) ++ min_objects = 4 * (fls(nr_cpu_ids) + 1); ++ max_objects = order_objects(slub_max_order, size, reserved); ++ min_objects = min(min_objects, max_objects); ++ ++ while (min_objects > 1) { ++ fraction = 16; ++ while (fraction >= 4) { ++ order = slab_order(size, min_objects, ++ slub_max_order, fraction, reserved); ++ if (order <= slub_max_order) ++ return order; ++ fraction /= 2; ++ } ++ min_objects--; ++ } ++ ++ /* ++ * We were unable to place multiple objects in a slab. Now ++ * lets see if we can place a single object there. ++ */ ++ order = slab_order(size, 1, slub_max_order, 1, reserved); ++ if (order <= slub_max_order) ++ return order; ++ ++ /* ++ * Doh this slab cannot be placed using slub_max_order. ++ */ ++ order = slab_order(size, 1, MAX_ORDER, 1, reserved); ++ if (order < MAX_ORDER) ++ return order; ++ return -ENOSYS; ++} ++ ++static void ++init_kmem_cache_node(struct kmem_cache_node *n) ++{ ++ n->nr_partial = 0; ++ spin_lock_init(&n->list_lock); ++ INIT_LIST_HEAD(&n->partial); ++#ifdef CONFIG_SLUB_DEBUG ++ atomic_long_set(&n->nr_slabs, 0); ++ atomic_long_set(&n->total_objects, 0); ++ INIT_LIST_HEAD(&n->full); ++#endif ++} ++ ++static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) ++{ ++ BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < ++ KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu)); ++ ++ /* ++ * Must align to double word boundary for the double cmpxchg ++ * instructions to work; see __pcpu_double_call_return_bool(). ++ */ ++ s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), ++ 2 * sizeof(void *)); ++ ++ if (!s->cpu_slab) ++ return 0; ++ ++ init_kmem_cache_cpus(s); ++ ++ return 1; ++} ++ ++static struct kmem_cache *kmem_cache_node; ++ ++/* ++ * No kmalloc_node yet so do it by hand. We know that this is the first ++ * slab on the node for this slabcache. There are no concurrent accesses ++ * possible. ++ * ++ * Note that this function only works on the kmem_cache_node ++ * when allocating for the kmem_cache_node. This is used for bootstrapping ++ * memory on a fresh node that has no slab structures yet. ++ */ ++static void early_kmem_cache_node_alloc(int node) ++{ ++ struct page *page; ++ struct kmem_cache_node *n; ++ ++ BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); ++ ++ page = new_slab(kmem_cache_node, GFP_NOWAIT, node); ++ ++ BUG_ON(!page); ++ if (page_to_nid(page) != node) { ++ pr_err("SLUB: Unable to allocate memory from node %d\n", node); ++ pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); ++ } ++ ++ n = page->freelist; ++ BUG_ON(!n); ++ page->freelist = get_freepointer(kmem_cache_node, n); ++ page->inuse = 1; ++ page->frozen = 0; ++ kmem_cache_node->node[node] = n; ++#ifdef CONFIG_SLUB_DEBUG ++ init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); ++ init_tracking(kmem_cache_node, n); ++#endif ++ kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node)); ++ init_kmem_cache_node(n); ++ inc_slabs_node(kmem_cache_node, node, page->objects); ++ ++ /* ++ * No locks need to be taken here as it has just been ++ * initialized and there is no concurrent access. ++ */ ++ __add_partial(n, page, DEACTIVATE_TO_HEAD); ++} ++ ++static void free_kmem_cache_nodes(struct kmem_cache *s) ++{ ++ int node; ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) { ++ kmem_cache_free(kmem_cache_node, n); ++ s->node[node] = NULL; ++ } ++} ++ ++static int init_kmem_cache_nodes(struct kmem_cache *s) ++{ ++ int node; ++ ++ for_each_node_state(node, N_NORMAL_MEMORY) { ++ struct kmem_cache_node *n; ++ ++ if (slab_state == DOWN) { ++ early_kmem_cache_node_alloc(node); ++ continue; ++ } ++ n = kmem_cache_alloc_node(kmem_cache_node, ++ GFP_KERNEL, node); ++ ++ if (!n) { ++ free_kmem_cache_nodes(s); ++ return 0; ++ } ++ ++ s->node[node] = n; ++ init_kmem_cache_node(n); ++ } ++ return 1; ++} ++ ++static void set_min_partial(struct kmem_cache *s, unsigned long min) ++{ ++ if (min < MIN_PARTIAL) ++ min = MIN_PARTIAL; ++ else if (min > MAX_PARTIAL) ++ min = MAX_PARTIAL; ++ s->min_partial = min; ++} ++ ++/* ++ * calculate_sizes() determines the order and the distribution of data within ++ * a slab object. ++ */ ++static int calculate_sizes(struct kmem_cache *s, int forced_order) ++{ ++ unsigned long flags = s->flags; ++ unsigned long size = s->object_size; ++ int order; ++ ++ /* ++ * Round up object size to the next word boundary. We can only ++ * place the free pointer at word boundaries and this determines ++ * the possible location of the free pointer. ++ */ ++ size = ALIGN(size, sizeof(void *)); ++ ++#ifdef CONFIG_SLUB_DEBUG ++ /* ++ * Determine if we can poison the object itself. If the user of ++ * the slab may touch the object after free or before allocation ++ * then we should never poison the object itself. ++ */ ++ if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && ++ !s->ctor) ++ s->flags |= __OBJECT_POISON; ++ else ++ s->flags &= ~__OBJECT_POISON; ++ ++ ++ /* ++ * If we are Redzoning then check if there is some space between the ++ * end of the object and the free pointer. If not then add an ++ * additional word to have some bytes to store Redzone information. ++ */ ++ if ((flags & SLAB_RED_ZONE) && size == s->object_size) ++ size += sizeof(void *); ++#endif ++ ++ /* ++ * With that we have determined the number of bytes in actual use ++ * by the object. This is the potential offset to the free pointer. ++ */ ++ s->inuse = size; ++ ++ if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || ++ s->ctor)) { ++ /* ++ * Relocate free pointer after the object if it is not ++ * permitted to overwrite the first word of the object on ++ * kmem_cache_free. ++ * ++ * This is the case if we do RCU, have a constructor or ++ * destructor or are poisoning the objects. ++ */ ++ s->offset = size; ++ size += sizeof(void *); ++ } ++ ++#ifdef CONFIG_SLUB_DEBUG ++ if (flags & SLAB_STORE_USER) ++ /* ++ * Need to store information about allocs and frees after ++ * the object. ++ */ ++ size += 2 * sizeof(struct track); ++ ++ if (flags & SLAB_RED_ZONE) ++ /* ++ * Add some empty padding so that we can catch ++ * overwrites from earlier objects rather than let ++ * tracking information or the free pointer be ++ * corrupted if a user writes before the start ++ * of the object. ++ */ ++ size += sizeof(void *); ++#endif ++ ++ /* ++ * SLUB stores one object immediately after another beginning from ++ * offset 0. In order to align the objects we have to simply size ++ * each object to conform to the alignment. ++ */ ++ size = ALIGN(size, s->align); ++ s->size = size; ++ if (forced_order >= 0) ++ order = forced_order; ++ else ++ order = calculate_order(size, s->reserved); ++ ++ if (order < 0) ++ return 0; ++ ++ s->allocflags = 0; ++ if (order) ++ s->allocflags |= __GFP_COMP; ++ ++ if (s->flags & SLAB_CACHE_DMA) ++ s->allocflags |= GFP_DMA; ++ ++ if (s->flags & SLAB_RECLAIM_ACCOUNT) ++ s->allocflags |= __GFP_RECLAIMABLE; ++ ++ /* ++ * Determine the number of objects per slab ++ */ ++ s->oo = oo_make(order, size, s->reserved); ++ s->min = oo_make(get_order(size), size, s->reserved); ++ if (oo_objects(s->oo) > oo_objects(s->max)) ++ s->max = s->oo; ++ ++ return !!oo_objects(s->oo); ++} ++ ++static int kmem_cache_open(struct kmem_cache *s, unsigned long flags) ++{ ++ s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor); ++ s->reserved = 0; ++ ++ if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU)) ++ s->reserved = sizeof(struct rcu_head); ++ ++ if (!calculate_sizes(s, -1)) ++ goto error; ++ if (disable_higher_order_debug) { ++ /* ++ * Disable debugging flags that store metadata if the min slab ++ * order increased. ++ */ ++ if (get_order(s->size) > get_order(s->object_size)) { ++ s->flags &= ~DEBUG_METADATA_FLAGS; ++ s->offset = 0; ++ if (!calculate_sizes(s, -1)) ++ goto error; ++ } ++ } ++ ++#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ ++ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) ++ if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0) ++ /* Enable fast mode */ ++ s->flags |= __CMPXCHG_DOUBLE; ++#endif ++ ++ /* ++ * The larger the object size is, the more pages we want on the partial ++ * list to avoid pounding the page allocator excessively. ++ */ ++ set_min_partial(s, ilog2(s->size) / 2); ++ ++ /* ++ * cpu_partial determined the maximum number of objects kept in the ++ * per cpu partial lists of a processor. ++ * ++ * Per cpu partial lists mainly contain slabs that just have one ++ * object freed. If they are used for allocation then they can be ++ * filled up again with minimal effort. The slab will never hit the ++ * per node partial lists and therefore no locking will be required. ++ * ++ * This setting also determines ++ * ++ * A) The number of objects from per cpu partial slabs dumped to the ++ * per node list when we reach the limit. ++ * B) The number of objects in cpu partial slabs to extract from the ++ * per node list when we run out of per cpu objects. We only fetch ++ * 50% to keep some capacity around for frees. ++ */ ++ if (!kmem_cache_has_cpu_partial(s)) ++ s->cpu_partial = 0; ++ else if (s->size >= PAGE_SIZE) ++ s->cpu_partial = 2; ++ else if (s->size >= 1024) ++ s->cpu_partial = 6; ++ else if (s->size >= 256) ++ s->cpu_partial = 13; ++ else ++ s->cpu_partial = 30; ++ ++#ifdef CONFIG_NUMA ++ s->remote_node_defrag_ratio = 1000; ++#endif ++ if (!init_kmem_cache_nodes(s)) ++ goto error; ++ ++ if (alloc_kmem_cache_cpus(s)) ++ return 0; ++ ++ free_kmem_cache_nodes(s); ++error: ++ if (flags & SLAB_PANIC) ++ panic("Cannot create slab %s size=%lu realsize=%u " ++ "order=%u offset=%u flags=%lx\n", ++ s->name, (unsigned long)s->size, s->size, ++ oo_order(s->oo), s->offset, flags); ++ return -EINVAL; ++} ++ ++static void list_slab_objects(struct kmem_cache *s, struct page *page, ++ const char *text) ++{ ++#ifdef CONFIG_SLUB_DEBUG ++ void *addr = page_address(page); ++ void *p; ++ unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) * ++ sizeof(long), GFP_ATOMIC); ++ if (!map) ++ return; ++ slab_err(s, page, text, s->name); ++ slab_lock(page); ++ ++ get_map(s, page, map); ++ for_each_object(p, s, addr, page->objects) { ++ ++ if (!test_bit(slab_index(p, s, addr), map)) { ++ pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr); ++ print_tracking(s, p); ++ } ++ } ++ slab_unlock(page); ++ kfree(map); ++#endif ++} ++ ++/* ++ * Attempt to free all partial slabs on a node. ++ * This is called from kmem_cache_close(). We must be the last thread ++ * using the cache and therefore we do not need to lock anymore. ++ */ ++static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) ++{ ++ struct page *page, *h; ++ ++ list_for_each_entry_safe(page, h, &n->partial, lru) { ++ if (!page->inuse) { ++ __remove_partial(n, page); ++ discard_slab(s, page); ++ } else { ++ list_slab_objects(s, page, ++ "Objects remaining in %s on kmem_cache_close()"); ++ } ++ } ++} ++ ++/* ++ * Release all resources used by a slab cache. ++ */ ++static inline int kmem_cache_close(struct kmem_cache *s) ++{ ++ int node; ++ struct kmem_cache_node *n; ++ ++ flush_all(s); ++ /* Attempt to free all objects */ ++ for_each_kmem_cache_node(s, node, n) { ++ free_partial(s, n); ++ if (n->nr_partial || slabs_node(s, node)) ++ return 1; ++ } ++ free_percpu(s->cpu_slab); ++ free_kmem_cache_nodes(s); ++ return 0; ++} ++ ++int __kmem_cache_shutdown(struct kmem_cache *s) ++{ ++ return kmem_cache_close(s); ++} ++ ++/******************************************************************** ++ * Kmalloc subsystem ++ *******************************************************************/ ++ ++static int __init setup_slub_min_order(char *str) ++{ ++ get_option(&str, &slub_min_order); ++ ++ return 1; ++} ++ ++__setup("slub_min_order=", setup_slub_min_order); ++ ++static int __init setup_slub_max_order(char *str) ++{ ++ get_option(&str, &slub_max_order); ++ slub_max_order = min(slub_max_order, MAX_ORDER - 1); ++ ++ return 1; ++} ++ ++__setup("slub_max_order=", setup_slub_max_order); ++ ++static int __init setup_slub_min_objects(char *str) ++{ ++ get_option(&str, &slub_min_objects); ++ ++ return 1; ++} ++ ++__setup("slub_min_objects=", setup_slub_min_objects); ++ ++void *__kmalloc(size_t size, gfp_t flags) ++{ ++ struct kmem_cache *s; ++ void *ret; ++ ++ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) ++ return kmalloc_large(size, flags); ++ ++ s = kmalloc_slab(size, flags); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(s))) ++ return s; ++ ++ ret = slab_alloc(s, flags, _RET_IP_); ++ ++ trace_kmalloc(_RET_IP_, ret, size, s->size, flags); ++ ++ kasan_kmalloc(s, ret, size); ++ ++ return ret; ++} ++EXPORT_SYMBOL(__kmalloc); ++ ++#ifdef CONFIG_NUMA ++static void *kmalloc_large_node(size_t size, gfp_t flags, int node) ++{ ++ struct page *page; ++ void *ptr = NULL; ++ ++ flags |= __GFP_COMP | __GFP_NOTRACK; ++ page = alloc_kmem_pages_node(node, flags, get_order(size)); ++ if (page) ++ ptr = page_address(page); ++ ++ kmalloc_large_node_hook(ptr, size, flags); ++ return ptr; ++} ++ ++void *__kmalloc_node(size_t size, gfp_t flags, int node) ++{ ++ struct kmem_cache *s; ++ void *ret; ++ ++ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { ++ ret = kmalloc_large_node(size, flags, node); ++ ++ trace_kmalloc_node(_RET_IP_, ret, ++ size, PAGE_SIZE << get_order(size), ++ flags, node); ++ ++ return ret; ++ } ++ ++ s = kmalloc_slab(size, flags); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(s))) ++ return s; ++ ++ ret = slab_alloc_node(s, flags, node, _RET_IP_); ++ ++ trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); ++ ++ kasan_kmalloc(s, ret, size); ++ ++ return ret; ++} ++EXPORT_SYMBOL(__kmalloc_node); ++#endif ++ ++static size_t __ksize(const void *object) ++{ ++ struct page *page; ++ ++ if (unlikely(object == ZERO_SIZE_PTR)) ++ return 0; ++ ++ page = virt_to_head_page(object); ++ ++ if (unlikely(!PageSlab(page))) { ++ WARN_ON(!PageCompound(page)); ++ return PAGE_SIZE << compound_order(page); ++ } ++ ++ return slab_ksize(page->slab_cache); ++} ++ ++size_t ksize(const void *object) ++{ ++ size_t size = __ksize(object); ++ /* We assume that ksize callers could use whole allocated area, ++ so we need unpoison this area. */ ++ kasan_krealloc(object, size); ++ return size; ++} ++EXPORT_SYMBOL(ksize); ++ ++void kfree(const void *x) ++{ ++ struct page *page; ++ void *object = (void *)x; ++ ++ trace_kfree(_RET_IP_, x); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(x))) ++ return; ++ ++ page = virt_to_head_page(x); ++ if (unlikely(!PageSlab(page))) { ++ BUG_ON(!PageCompound(page)); ++ kfree_hook(x); ++ __free_kmem_pages(page, compound_order(page)); ++ return; ++ } ++ slab_free(page->slab_cache, page, object, _RET_IP_); ++} ++EXPORT_SYMBOL(kfree); ++ ++#define SHRINK_PROMOTE_MAX 32 ++ ++/* ++ * kmem_cache_shrink discards empty slabs and promotes the slabs filled ++ * up most to the head of the partial lists. New allocations will then ++ * fill those up and thus they can be removed from the partial lists. ++ * ++ * The slabs with the least items are placed last. This results in them ++ * being allocated from last increasing the chance that the last objects ++ * are freed in them. ++ */ ++int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate) ++{ ++ int node; ++ int i; ++ struct kmem_cache_node *n; ++ struct page *page; ++ struct page *t; ++ struct list_head discard; ++ struct list_head promote[SHRINK_PROMOTE_MAX]; ++ unsigned long flags; ++ int ret = 0; ++ ++ if (deactivate) { ++ /* ++ * Disable empty slabs caching. Used to avoid pinning offline ++ * memory cgroups by kmem pages that can be freed. ++ */ ++ s->cpu_partial = 0; ++ s->min_partial = 0; ++ ++ /* ++ * s->cpu_partial is checked locklessly (see put_cpu_partial), ++ * so we have to make sure the change is visible. ++ */ ++ kick_all_cpus_sync(); ++ } ++ ++ flush_all(s); ++ for_each_kmem_cache_node(s, node, n) { ++ INIT_LIST_HEAD(&discard); ++ for (i = 0; i < SHRINK_PROMOTE_MAX; i++) ++ INIT_LIST_HEAD(promote + i); ++ ++ spin_lock_irqsave(&n->list_lock, flags); ++ ++ /* ++ * Build lists of slabs to discard or promote. ++ * ++ * Note that concurrent frees may occur while we hold the ++ * list_lock. page->inuse here is the upper limit. ++ */ ++ list_for_each_entry_safe(page, t, &n->partial, lru) { ++ int free = page->objects - page->inuse; ++ ++ /* Do not reread page->inuse */ ++ barrier(); ++ ++ /* We do not keep full slabs on the list */ ++ BUG_ON(free <= 0); ++ ++ if (free == page->objects) { ++ list_move(&page->lru, &discard); ++ n->nr_partial--; ++ } else if (free <= SHRINK_PROMOTE_MAX) ++ list_move(&page->lru, promote + free - 1); ++ } ++ ++ /* ++ * Promote the slabs filled up most to the head of the ++ * partial list. ++ */ ++ for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) ++ list_splice(promote + i, &n->partial); ++ ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ ++ /* Release empty slabs */ ++ list_for_each_entry_safe(page, t, &discard, lru) ++ discard_slab(s, page); ++ ++ if (slabs_node(s, node)) ++ ret = 1; ++ } ++ ++ return ret; ++} ++ ++static int slab_mem_going_offline_callback(void *arg) ++{ ++ struct kmem_cache *s; ++ ++ mutex_lock(&slab_mutex); ++ list_for_each_entry(s, &slab_caches, list) ++ __kmem_cache_shrink(s, false); ++ mutex_unlock(&slab_mutex); ++ ++ return 0; ++} ++ ++static void slab_mem_offline_callback(void *arg) ++{ ++ struct kmem_cache_node *n; ++ struct kmem_cache *s; ++ struct memory_notify *marg = arg; ++ int offline_node; ++ ++ offline_node = marg->status_change_nid_normal; ++ ++ /* ++ * If the node still has available memory. we need kmem_cache_node ++ * for it yet. ++ */ ++ if (offline_node < 0) ++ return; ++ ++ mutex_lock(&slab_mutex); ++ list_for_each_entry(s, &slab_caches, list) { ++ n = get_node(s, offline_node); ++ if (n) { ++ /* ++ * if n->nr_slabs > 0, slabs still exist on the node ++ * that is going down. We were unable to free them, ++ * and offline_pages() function shouldn't call this ++ * callback. So, we must fail. ++ */ ++ BUG_ON(slabs_node(s, offline_node)); ++ ++ s->node[offline_node] = NULL; ++ kmem_cache_free(kmem_cache_node, n); ++ } ++ } ++ mutex_unlock(&slab_mutex); ++} ++ ++static int slab_mem_going_online_callback(void *arg) ++{ ++ struct kmem_cache_node *n; ++ struct kmem_cache *s; ++ struct memory_notify *marg = arg; ++ int nid = marg->status_change_nid_normal; ++ int ret = 0; ++ ++ /* ++ * If the node's memory is already available, then kmem_cache_node is ++ * already created. Nothing to do. ++ */ ++ if (nid < 0) ++ return 0; ++ ++ /* ++ * We are bringing a node online. No memory is available yet. We must ++ * allocate a kmem_cache_node structure in order to bring the node ++ * online. ++ */ ++ mutex_lock(&slab_mutex); ++ list_for_each_entry(s, &slab_caches, list) { ++ /* ++ * XXX: kmem_cache_alloc_node will fallback to other nodes ++ * since memory is not yet available from the node that ++ * is brought up. ++ */ ++ n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL); ++ if (!n) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ init_kmem_cache_node(n); ++ s->node[nid] = n; ++ } ++out: ++ mutex_unlock(&slab_mutex); ++ return ret; ++} ++ ++static int slab_memory_callback(struct notifier_block *self, ++ unsigned long action, void *arg) ++{ ++ int ret = 0; ++ ++ switch (action) { ++ case MEM_GOING_ONLINE: ++ ret = slab_mem_going_online_callback(arg); ++ break; ++ case MEM_GOING_OFFLINE: ++ ret = slab_mem_going_offline_callback(arg); ++ break; ++ case MEM_OFFLINE: ++ case MEM_CANCEL_ONLINE: ++ slab_mem_offline_callback(arg); ++ break; ++ case MEM_ONLINE: ++ case MEM_CANCEL_OFFLINE: ++ break; ++ } ++ if (ret) ++ ret = notifier_from_errno(ret); ++ else ++ ret = NOTIFY_OK; ++ return ret; ++} ++ ++static struct notifier_block slab_memory_callback_nb = { ++ .notifier_call = slab_memory_callback, ++ .priority = SLAB_CALLBACK_PRI, ++}; ++ ++/******************************************************************** ++ * Basic setup of slabs ++ *******************************************************************/ ++ ++/* ++ * Used for early kmem_cache structures that were allocated using ++ * the page allocator. Allocate them properly then fix up the pointers ++ * that may be pointing to the wrong kmem_cache structure. ++ */ ++ ++static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) ++{ ++ int node; ++ struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); ++ struct kmem_cache_node *n; ++ ++ memcpy(s, static_cache, kmem_cache->object_size); ++ ++ /* ++ * This runs very early, and only the boot processor is supposed to be ++ * up. Even if it weren't true, IRQs are not up so we couldn't fire ++ * IPIs around. ++ */ ++ __flush_cpu_slab(s, smp_processor_id()); ++ for_each_kmem_cache_node(s, node, n) { ++ struct page *p; ++ ++ list_for_each_entry(p, &n->partial, lru) ++ p->slab_cache = s; ++ ++#ifdef CONFIG_SLUB_DEBUG ++ list_for_each_entry(p, &n->full, lru) ++ p->slab_cache = s; ++#endif ++ } ++ slab_init_memcg_params(s); ++ list_add(&s->list, &slab_caches); ++ return s; ++} ++ ++void __init kmem_cache_init(void) ++{ ++ static __initdata struct kmem_cache boot_kmem_cache, ++ boot_kmem_cache_node; ++ ++ if (debug_guardpage_minorder()) ++ slub_max_order = 0; ++ ++ kmem_cache_node = &boot_kmem_cache_node; ++ kmem_cache = &boot_kmem_cache; ++ ++ create_boot_cache(kmem_cache_node, "kmem_cache_node", ++ sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN); ++ ++ register_hotmemory_notifier(&slab_memory_callback_nb); ++ ++ /* Able to allocate the per node structures */ ++ slab_state = PARTIAL; ++ ++ create_boot_cache(kmem_cache, "kmem_cache", ++ offsetof(struct kmem_cache, node) + ++ nr_node_ids * sizeof(struct kmem_cache_node *), ++ SLAB_HWCACHE_ALIGN); ++ ++ kmem_cache = bootstrap(&boot_kmem_cache); ++ ++ /* ++ * Allocate kmem_cache_node properly from the kmem_cache slab. ++ * kmem_cache_node is separately allocated so no need to ++ * update any list pointers. ++ */ ++ kmem_cache_node = bootstrap(&boot_kmem_cache_node); ++ ++ /* Now we can use the kmem_cache to allocate kmalloc slabs */ ++ create_kmalloc_caches(0); ++ ++#ifdef CONFIG_SMP ++ register_cpu_notifier(&slab_notifier); ++#endif ++ ++ pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n", ++ cache_line_size(), ++ slub_min_order, slub_max_order, slub_min_objects, ++ nr_cpu_ids, nr_node_ids); ++} ++ ++void __init kmem_cache_init_late(void) ++{ ++} ++ ++struct kmem_cache * ++__kmem_cache_alias(const char *name, size_t size, size_t align, ++ unsigned long flags, void (*ctor)(void *)) ++{ ++ struct kmem_cache *s, *c; ++ ++ s = find_mergeable(size, align, flags, name, ctor); ++ if (s) { ++ s->refcount++; ++ ++ /* ++ * Adjust the object sizes so that we clear ++ * the complete object on kzalloc. ++ */ ++ s->object_size = max(s->object_size, (int)size); ++ s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); ++ ++ for_each_memcg_cache(c, s) { ++ c->object_size = s->object_size; ++ c->inuse = max_t(int, c->inuse, ++ ALIGN(size, sizeof(void *))); ++ } ++ ++ if (sysfs_slab_alias(s, name)) { ++ s->refcount--; ++ s = NULL; ++ } ++ } ++ ++ return s; ++} ++ ++int __kmem_cache_create(struct kmem_cache *s, unsigned long flags) ++{ ++ int err; ++ ++ err = kmem_cache_open(s, flags); ++ if (err) ++ return err; ++ ++ /* Mutex is not taken during early boot */ ++ if (slab_state <= UP) ++ return 0; ++ ++ memcg_propagate_slab_attrs(s); ++ err = sysfs_slab_add(s); ++ if (err) ++ kmem_cache_close(s); ++ ++ return err; ++} ++ ++#ifdef CONFIG_SMP ++/* ++ * Use the cpu notifier to insure that the cpu slabs are flushed when ++ * necessary. ++ */ ++static int slab_cpuup_callback(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) ++{ ++ long cpu = (long)hcpu; ++ struct kmem_cache *s; ++ unsigned long flags; ++ ++ switch (action) { ++ case CPU_UP_CANCELED: ++ case CPU_UP_CANCELED_FROZEN: ++ case CPU_DEAD: ++ case CPU_DEAD_FROZEN: ++ mutex_lock(&slab_mutex); ++ list_for_each_entry(s, &slab_caches, list) { ++ local_irq_save(flags); ++ __flush_cpu_slab(s, cpu); ++ local_irq_restore(flags); ++ } ++ mutex_unlock(&slab_mutex); ++ break; ++ default: ++ break; ++ } ++ return NOTIFY_OK; ++} ++ ++static struct notifier_block slab_notifier = { ++ .notifier_call = slab_cpuup_callback ++}; ++ ++#endif ++ ++void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) ++{ ++ struct kmem_cache *s; ++ void *ret; ++ ++ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) ++ return kmalloc_large(size, gfpflags); ++ ++ s = kmalloc_slab(size, gfpflags); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(s))) ++ return s; ++ ++ ret = slab_alloc(s, gfpflags, caller); ++ ++ /* Honor the call site pointer we received. */ ++ trace_kmalloc(caller, ret, size, s->size, gfpflags); ++ ++ return ret; ++} ++ ++#ifdef CONFIG_NUMA ++void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, ++ int node, unsigned long caller) ++{ ++ struct kmem_cache *s; ++ void *ret; ++ ++ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { ++ ret = kmalloc_large_node(size, gfpflags, node); ++ ++ trace_kmalloc_node(caller, ret, ++ size, PAGE_SIZE << get_order(size), ++ gfpflags, node); ++ ++ return ret; ++ } ++ ++ s = kmalloc_slab(size, gfpflags); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(s))) ++ return s; ++ ++ ret = slab_alloc_node(s, gfpflags, node, caller); ++ ++ /* Honor the call site pointer we received. */ ++ trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); ++ ++ return ret; ++} ++#endif ++ ++#ifdef CONFIG_SYSFS ++static int count_inuse(struct page *page) ++{ ++ return page->inuse; ++} ++ ++static int count_total(struct page *page) ++{ ++ return page->objects; ++} ++#endif ++ ++#ifdef CONFIG_SLUB_DEBUG ++static int validate_slab(struct kmem_cache *s, struct page *page, ++ unsigned long *map) ++{ ++ void *p; ++ void *addr = page_address(page); ++ ++ if (!check_slab(s, page) || ++ !on_freelist(s, page, NULL)) ++ return 0; ++ ++ /* Now we know that a valid freelist exists */ ++ bitmap_zero(map, page->objects); ++ ++ get_map(s, page, map); ++ for_each_object(p, s, addr, page->objects) { ++ if (test_bit(slab_index(p, s, addr), map)) ++ if (!check_object(s, page, p, SLUB_RED_INACTIVE)) ++ return 0; ++ } ++ ++ for_each_object(p, s, addr, page->objects) ++ if (!test_bit(slab_index(p, s, addr), map)) ++ if (!check_object(s, page, p, SLUB_RED_ACTIVE)) ++ return 0; ++ return 1; ++} ++ ++static void validate_slab_slab(struct kmem_cache *s, struct page *page, ++ unsigned long *map) ++{ ++ slab_lock(page); ++ validate_slab(s, page, map); ++ slab_unlock(page); ++} ++ ++static int validate_slab_node(struct kmem_cache *s, ++ struct kmem_cache_node *n, unsigned long *map) ++{ ++ unsigned long count = 0; ++ struct page *page; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&n->list_lock, flags); ++ ++ list_for_each_entry(page, &n->partial, lru) { ++ validate_slab_slab(s, page, map); ++ count++; ++ } ++ if (count != n->nr_partial) ++ pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n", ++ s->name, count, n->nr_partial); ++ ++ if (!(s->flags & SLAB_STORE_USER)) ++ goto out; ++ ++ list_for_each_entry(page, &n->full, lru) { ++ validate_slab_slab(s, page, map); ++ count++; ++ } ++ if (count != atomic_long_read(&n->nr_slabs)) ++ pr_err("SLUB: %s %ld slabs counted but counter=%ld\n", ++ s->name, count, atomic_long_read(&n->nr_slabs)); ++ ++out: ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ return count; ++} ++ ++static long validate_slab_cache(struct kmem_cache *s) ++{ ++ int node; ++ unsigned long count = 0; ++ unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * ++ sizeof(unsigned long), GFP_KERNEL); ++ struct kmem_cache_node *n; ++ ++ if (!map) ++ return -ENOMEM; ++ ++ flush_all(s); ++ for_each_kmem_cache_node(s, node, n) ++ count += validate_slab_node(s, n, map); ++ kfree(map); ++ return count; ++} ++/* ++ * Generate lists of code addresses where slabcache objects are allocated ++ * and freed. ++ */ ++ ++struct location { ++ unsigned long count; ++ unsigned long addr; ++ long long sum_time; ++ long min_time; ++ long max_time; ++ long min_pid; ++ long max_pid; ++ DECLARE_BITMAP(cpus, NR_CPUS); ++ nodemask_t nodes; ++}; ++ ++struct loc_track { ++ unsigned long max; ++ unsigned long count; ++ struct location *loc; ++}; ++ ++static void free_loc_track(struct loc_track *t) ++{ ++ if (t->max) ++ free_pages((unsigned long)t->loc, ++ get_order(sizeof(struct location) * t->max)); ++} ++ ++static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) ++{ ++ struct location *l; ++ int order; ++ ++ order = get_order(sizeof(struct location) * max); ++ ++ l = (void *)__get_free_pages(flags, order); ++ if (!l) ++ return 0; ++ ++ if (t->count) { ++ memcpy(l, t->loc, sizeof(struct location) * t->count); ++ free_loc_track(t); ++ } ++ t->max = max; ++ t->loc = l; ++ return 1; ++} ++ ++static int add_location(struct loc_track *t, struct kmem_cache *s, ++ const struct track *track) ++{ ++ long start, end, pos; ++ struct location *l; ++ unsigned long caddr; ++ unsigned long age = jiffies - track->when; ++ ++ start = -1; ++ end = t->count; ++ ++ for ( ; ; ) { ++ pos = start + (end - start + 1) / 2; ++ ++ /* ++ * There is nothing at "end". If we end up there ++ * we need to add something to before end. ++ */ ++ if (pos == end) ++ break; ++ ++ caddr = t->loc[pos].addr; ++ if (track->addr == caddr) { ++ ++ l = &t->loc[pos]; ++ l->count++; ++ if (track->when) { ++ l->sum_time += age; ++ if (age < l->min_time) ++ l->min_time = age; ++ if (age > l->max_time) ++ l->max_time = age; ++ ++ if (track->pid < l->min_pid) ++ l->min_pid = track->pid; ++ if (track->pid > l->max_pid) ++ l->max_pid = track->pid; ++ ++ cpumask_set_cpu(track->cpu, ++ to_cpumask(l->cpus)); ++ } ++ node_set(page_to_nid(virt_to_page(track)), l->nodes); ++ return 1; ++ } ++ ++ if (track->addr < caddr) ++ end = pos; ++ else ++ start = pos; ++ } ++ ++ /* ++ * Not found. Insert new tracking element. ++ */ ++ if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) ++ return 0; ++ ++ l = t->loc + pos; ++ if (pos < t->count) ++ memmove(l + 1, l, ++ (t->count - pos) * sizeof(struct location)); ++ t->count++; ++ l->count = 1; ++ l->addr = track->addr; ++ l->sum_time = age; ++ l->min_time = age; ++ l->max_time = age; ++ l->min_pid = track->pid; ++ l->max_pid = track->pid; ++ cpumask_clear(to_cpumask(l->cpus)); ++ cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); ++ nodes_clear(l->nodes); ++ node_set(page_to_nid(virt_to_page(track)), l->nodes); ++ return 1; ++} ++ ++static void process_slab(struct loc_track *t, struct kmem_cache *s, ++ struct page *page, enum track_item alloc, ++ unsigned long *map) ++{ ++ void *addr = page_address(page); ++ void *p; ++ ++ bitmap_zero(map, page->objects); ++ get_map(s, page, map); ++ ++ for_each_object(p, s, addr, page->objects) ++ if (!test_bit(slab_index(p, s, addr), map)) ++ add_location(t, s, get_track(s, p, alloc)); ++} ++ ++static int list_locations(struct kmem_cache *s, char *buf, ++ enum track_item alloc) ++{ ++ int len = 0; ++ unsigned long i; ++ struct loc_track t = { 0, 0, NULL }; ++ int node; ++ unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * ++ sizeof(unsigned long), GFP_KERNEL); ++ struct kmem_cache_node *n; ++ ++ if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), ++ GFP_TEMPORARY)) { ++ kfree(map); ++ return sprintf(buf, "Out of memory\n"); ++ } ++ /* Push back cpu slabs */ ++ flush_all(s); ++ ++ for_each_kmem_cache_node(s, node, n) { ++ unsigned long flags; ++ struct page *page; ++ ++ if (!atomic_long_read(&n->nr_slabs)) ++ continue; ++ ++ spin_lock_irqsave(&n->list_lock, flags); ++ list_for_each_entry(page, &n->partial, lru) ++ process_slab(&t, s, page, alloc, map); ++ list_for_each_entry(page, &n->full, lru) ++ process_slab(&t, s, page, alloc, map); ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ } ++ ++ for (i = 0; i < t.count; i++) { ++ struct location *l = &t.loc[i]; ++ ++ if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100) ++ break; ++ len += sprintf(buf + len, "%7ld ", l->count); ++ ++ if (l->addr) ++ len += sprintf(buf + len, "%pS", (void *)l->addr); ++ else ++ len += sprintf(buf + len, "<not-available>"); ++ ++ if (l->sum_time != l->min_time) { ++ len += sprintf(buf + len, " age=%ld/%ld/%ld", ++ l->min_time, ++ (long)div_u64(l->sum_time, l->count), ++ l->max_time); ++ } else ++ len += sprintf(buf + len, " age=%ld", ++ l->min_time); ++ ++ if (l->min_pid != l->max_pid) ++ len += sprintf(buf + len, " pid=%ld-%ld", ++ l->min_pid, l->max_pid); ++ else ++ len += sprintf(buf + len, " pid=%ld", ++ l->min_pid); ++ ++ if (num_online_cpus() > 1 && ++ !cpumask_empty(to_cpumask(l->cpus)) && ++ len < PAGE_SIZE - 60) ++ len += scnprintf(buf + len, PAGE_SIZE - len - 50, ++ " cpus=%*pbl", ++ cpumask_pr_args(to_cpumask(l->cpus))); ++ ++ if (nr_online_nodes > 1 && !nodes_empty(l->nodes) && ++ len < PAGE_SIZE - 60) ++ len += scnprintf(buf + len, PAGE_SIZE - len - 50, ++ " nodes=%*pbl", ++ nodemask_pr_args(&l->nodes)); ++ ++ len += sprintf(buf + len, "\n"); ++ } ++ ++ free_loc_track(&t); ++ kfree(map); ++ if (!t.count) ++ len += sprintf(buf, "No data\n"); ++ return len; ++} ++#endif ++ ++#ifdef SLUB_RESILIENCY_TEST ++static void __init resiliency_test(void) ++{ ++ u8 *p; ++ ++ BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10); ++ ++ pr_err("SLUB resiliency testing\n"); ++ pr_err("-----------------------\n"); ++ pr_err("A. Corruption after allocation\n"); ++ ++ p = kzalloc(16, GFP_KERNEL); ++ p[16] = 0x12; ++ pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n", ++ p + 16); ++ ++ validate_slab_cache(kmalloc_caches[4]); ++ ++ /* Hmmm... The next two are dangerous */ ++ p = kzalloc(32, GFP_KERNEL); ++ p[32 + sizeof(void *)] = 0x34; ++ pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n", ++ p); ++ pr_err("If allocated object is overwritten then not detectable\n\n"); ++ ++ validate_slab_cache(kmalloc_caches[5]); ++ p = kzalloc(64, GFP_KERNEL); ++ p += 64 + (get_cycles() & 0xff) * sizeof(void *); ++ *p = 0x56; ++ pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n", ++ p); ++ pr_err("If allocated object is overwritten then not detectable\n\n"); ++ validate_slab_cache(kmalloc_caches[6]); ++ ++ pr_err("\nB. Corruption after free\n"); ++ p = kzalloc(128, GFP_KERNEL); ++ kfree(p); ++ *p = 0x78; ++ pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p); ++ validate_slab_cache(kmalloc_caches[7]); ++ ++ p = kzalloc(256, GFP_KERNEL); ++ kfree(p); ++ p[50] = 0x9a; ++ pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p); ++ validate_slab_cache(kmalloc_caches[8]); ++ ++ p = kzalloc(512, GFP_KERNEL); ++ kfree(p); ++ p[512] = 0xab; ++ pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p); ++ validate_slab_cache(kmalloc_caches[9]); ++} ++#else ++#ifdef CONFIG_SYSFS ++static void resiliency_test(void) {}; ++#endif ++#endif ++ ++#ifdef CONFIG_SYSFS ++enum slab_stat_type { ++ SL_ALL, /* All slabs */ ++ SL_PARTIAL, /* Only partially allocated slabs */ ++ SL_CPU, /* Only slabs used for cpu caches */ ++ SL_OBJECTS, /* Determine allocated objects not slabs */ ++ SL_TOTAL /* Determine object capacity not slabs */ ++}; ++ ++#define SO_ALL (1 << SL_ALL) ++#define SO_PARTIAL (1 << SL_PARTIAL) ++#define SO_CPU (1 << SL_CPU) ++#define SO_OBJECTS (1 << SL_OBJECTS) ++#define SO_TOTAL (1 << SL_TOTAL) ++ ++static ssize_t show_slab_objects(struct kmem_cache *s, ++ char *buf, unsigned long flags) ++{ ++ unsigned long total = 0; ++ int node; ++ int x; ++ unsigned long *nodes; ++ ++ nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL); ++ if (!nodes) ++ return -ENOMEM; ++ ++ if (flags & SO_CPU) { ++ int cpu; ++ ++ for_each_possible_cpu(cpu) { ++ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, ++ cpu); ++ int node; ++ struct page *page; ++ ++ page = READ_ONCE(c->page); ++ if (!page) ++ continue; ++ ++ node = page_to_nid(page); ++ if (flags & SO_TOTAL) ++ x = page->objects; ++ else if (flags & SO_OBJECTS) ++ x = page->inuse; ++ else ++ x = 1; ++ ++ total += x; ++ nodes[node] += x; ++ ++ page = READ_ONCE(c->partial); ++ if (page) { ++ node = page_to_nid(page); ++ if (flags & SO_TOTAL) ++ WARN_ON_ONCE(1); ++ else if (flags & SO_OBJECTS) ++ WARN_ON_ONCE(1); ++ else ++ x = page->pages; ++ total += x; ++ nodes[node] += x; ++ } ++ } ++ } ++ ++ get_online_mems(); ++#ifdef CONFIG_SLUB_DEBUG ++ if (flags & SO_ALL) { ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) { ++ ++ if (flags & SO_TOTAL) ++ x = atomic_long_read(&n->total_objects); ++ else if (flags & SO_OBJECTS) ++ x = atomic_long_read(&n->total_objects) - ++ count_partial(n, count_free); ++ else ++ x = atomic_long_read(&n->nr_slabs); ++ total += x; ++ nodes[node] += x; ++ } ++ ++ } else ++#endif ++ if (flags & SO_PARTIAL) { ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) { ++ if (flags & SO_TOTAL) ++ x = count_partial(n, count_total); ++ else if (flags & SO_OBJECTS) ++ x = count_partial(n, count_inuse); ++ else ++ x = n->nr_partial; ++ total += x; ++ nodes[node] += x; ++ } ++ } ++ x = sprintf(buf, "%lu", total); ++#ifdef CONFIG_NUMA ++ for (node = 0; node < nr_node_ids; node++) ++ if (nodes[node]) ++ x += sprintf(buf + x, " N%d=%lu", ++ node, nodes[node]); ++#endif ++ put_online_mems(); ++ kfree(nodes); ++ return x + sprintf(buf + x, "\n"); ++} ++ ++#ifdef CONFIG_SLUB_DEBUG ++static int any_slab_objects(struct kmem_cache *s) ++{ ++ int node; ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) ++ if (atomic_long_read(&n->total_objects)) ++ return 1; ++ ++ return 0; ++} ++#endif ++ ++#define to_slab_attr(n) container_of(n, struct slab_attribute, attr) ++#define to_slab(n) container_of(n, struct kmem_cache, kobj) ++ ++struct slab_attribute { ++ struct attribute attr; ++ ssize_t (*show)(struct kmem_cache *s, char *buf); ++ ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); ++}; ++ ++#define SLAB_ATTR_RO(_name) \ ++ static struct slab_attribute _name##_attr = \ ++ __ATTR(_name, 0400, _name##_show, NULL) ++ ++#define SLAB_ATTR(_name) \ ++ static struct slab_attribute _name##_attr = \ ++ __ATTR(_name, 0600, _name##_show, _name##_store) ++ ++static ssize_t slab_size_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->size); ++} ++SLAB_ATTR_RO(slab_size); ++ ++static ssize_t align_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->align); ++} ++SLAB_ATTR_RO(align); ++ ++static ssize_t object_size_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->object_size); ++} ++SLAB_ATTR_RO(object_size); ++ ++static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", oo_objects(s->oo)); ++} ++SLAB_ATTR_RO(objs_per_slab); ++ ++static ssize_t order_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ unsigned long order; ++ int err; ++ ++ err = kstrtoul(buf, 10, &order); ++ if (err) ++ return err; ++ ++ if (order > slub_max_order || order < slub_min_order) ++ return -EINVAL; ++ ++ calculate_sizes(s, order); ++ return length; ++} ++ ++static ssize_t order_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", oo_order(s->oo)); ++} ++SLAB_ATTR(order); ++ ++static ssize_t min_partial_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%lu\n", s->min_partial); ++} ++ ++static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, ++ size_t length) ++{ ++ unsigned long min; ++ int err; ++ ++ err = kstrtoul(buf, 10, &min); ++ if (err) ++ return err; ++ ++ set_min_partial(s, min); ++ return length; ++} ++SLAB_ATTR(min_partial); ++ ++static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%u\n", s->cpu_partial); ++} ++ ++static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, ++ size_t length) ++{ ++ unsigned long objects; ++ int err; ++ ++ err = kstrtoul(buf, 10, &objects); ++ if (err) ++ return err; ++ if (objects && !kmem_cache_has_cpu_partial(s)) ++ return -EINVAL; ++ ++ s->cpu_partial = objects; ++ flush_all(s); ++ return length; ++} ++SLAB_ATTR(cpu_partial); ++ ++static ssize_t ctor_show(struct kmem_cache *s, char *buf) ++{ ++ if (!s->ctor) ++ return 0; ++ return sprintf(buf, "%pS\n", s->ctor); ++} ++SLAB_ATTR_RO(ctor); ++ ++static ssize_t aliases_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); ++} ++SLAB_ATTR_RO(aliases); ++ ++static ssize_t partial_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_PARTIAL); ++} ++SLAB_ATTR_RO(partial); ++ ++static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_CPU); ++} ++SLAB_ATTR_RO(cpu_slabs); ++ ++static ssize_t objects_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS); ++} ++SLAB_ATTR_RO(objects); ++ ++static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS); ++} ++SLAB_ATTR_RO(objects_partial); ++ ++static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) ++{ ++ int objects = 0; ++ int pages = 0; ++ int cpu; ++ int len; ++ ++ for_each_online_cpu(cpu) { ++ struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial; ++ ++ if (page) { ++ pages += page->pages; ++ objects += page->pobjects; ++ } ++ } ++ ++ len = sprintf(buf, "%d(%d)", objects, pages); ++ ++#ifdef CONFIG_SMP ++ for_each_online_cpu(cpu) { ++ struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial; ++ ++ if (page && len < PAGE_SIZE - 20) ++ len += sprintf(buf + len, " C%d=%d(%d)", cpu, ++ page->pobjects, page->pages); ++ } ++#endif ++ return len + sprintf(buf + len, "\n"); ++} ++SLAB_ATTR_RO(slabs_cpu_partial); ++ ++static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); ++} ++ ++static ssize_t reclaim_account_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ s->flags &= ~SLAB_RECLAIM_ACCOUNT; ++ if (buf[0] == '1') ++ s->flags |= SLAB_RECLAIM_ACCOUNT; ++ return length; ++} ++SLAB_ATTR(reclaim_account); ++ ++static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); ++} ++SLAB_ATTR_RO(hwcache_align); ++ ++#ifdef CONFIG_ZONE_DMA ++static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); ++} ++SLAB_ATTR_RO(cache_dma); ++#endif ++ ++static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU)); ++} ++SLAB_ATTR_RO(destroy_by_rcu); ++ ++static ssize_t reserved_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->reserved); ++} ++SLAB_ATTR_RO(reserved); ++ ++#ifdef CONFIG_SLUB_DEBUG ++static ssize_t slabs_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_ALL); ++} ++SLAB_ATTR_RO(slabs); ++ ++static ssize_t total_objects_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); ++} ++SLAB_ATTR_RO(total_objects); ++ ++static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE)); ++} ++ ++static ssize_t sanity_checks_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ s->flags &= ~SLAB_DEBUG_FREE; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_DEBUG_FREE; ++ } ++ return length; ++} ++SLAB_ATTR(sanity_checks); ++ ++static ssize_t trace_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE)); ++} ++ ++static ssize_t trace_store(struct kmem_cache *s, const char *buf, ++ size_t length) ++{ ++ /* ++ * Tracing a merged cache is going to give confusing results ++ * as well as cause other issues like converting a mergeable ++ * cache into an umergeable one. ++ */ ++ if (s->refcount > 1) ++ return -EINVAL; ++ ++ s->flags &= ~SLAB_TRACE; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_TRACE; ++ } ++ return length; ++} ++SLAB_ATTR(trace); ++ ++static ssize_t red_zone_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); ++} ++ ++static ssize_t red_zone_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ if (any_slab_objects(s)) ++ return -EBUSY; ++ ++ s->flags &= ~SLAB_RED_ZONE; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_RED_ZONE; ++ } ++ calculate_sizes(s, -1); ++ return length; ++} ++SLAB_ATTR(red_zone); ++ ++static ssize_t poison_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON)); ++} ++ ++static ssize_t poison_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ if (any_slab_objects(s)) ++ return -EBUSY; ++ ++ s->flags &= ~SLAB_POISON; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_POISON; ++ } ++ calculate_sizes(s, -1); ++ return length; ++} ++SLAB_ATTR(poison); ++ ++static ssize_t store_user_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); ++} ++ ++static ssize_t store_user_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ if (any_slab_objects(s)) ++ return -EBUSY; ++ ++ s->flags &= ~SLAB_STORE_USER; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_STORE_USER; ++ } ++ calculate_sizes(s, -1); ++ return length; ++} ++SLAB_ATTR(store_user); ++ ++static ssize_t validate_show(struct kmem_cache *s, char *buf) ++{ ++ return 0; ++} ++ ++static ssize_t validate_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ int ret = -EINVAL; ++ ++ if (buf[0] == '1') { ++ ret = validate_slab_cache(s); ++ if (ret >= 0) ++ ret = length; ++ } ++ return ret; ++} ++SLAB_ATTR(validate); ++ ++static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return -ENOSYS; ++ return list_locations(s, buf, TRACK_ALLOC); ++} ++SLAB_ATTR_RO(alloc_calls); ++ ++static ssize_t free_calls_show(struct kmem_cache *s, char *buf) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return -ENOSYS; ++ return list_locations(s, buf, TRACK_FREE); ++} ++SLAB_ATTR_RO(free_calls); ++#endif /* CONFIG_SLUB_DEBUG */ ++ ++#ifdef CONFIG_FAILSLAB ++static ssize_t failslab_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); ++} ++ ++static ssize_t failslab_store(struct kmem_cache *s, const char *buf, ++ size_t length) ++{ ++ if (s->refcount > 1) ++ return -EINVAL; ++ ++ s->flags &= ~SLAB_FAILSLAB; ++ if (buf[0] == '1') ++ s->flags |= SLAB_FAILSLAB; ++ return length; ++} ++SLAB_ATTR(failslab); ++#endif ++ ++static ssize_t shrink_show(struct kmem_cache *s, char *buf) ++{ ++ return 0; ++} ++ ++static ssize_t shrink_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ if (buf[0] == '1') ++ kmem_cache_shrink(s); ++ else ++ return -EINVAL; ++ return length; ++} ++SLAB_ATTR(shrink); ++ ++#ifdef CONFIG_NUMA ++static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10); ++} ++ ++static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ unsigned long ratio; ++ int err; ++ ++ err = kstrtoul(buf, 10, &ratio); ++ if (err) ++ return err; ++ ++ if (ratio <= 100) ++ s->remote_node_defrag_ratio = ratio * 10; ++ ++ return length; ++} ++SLAB_ATTR(remote_node_defrag_ratio); ++#endif ++ ++#ifdef CONFIG_SLUB_STATS ++static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) ++{ ++ unsigned long sum = 0; ++ int cpu; ++ int len; ++ int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL); ++ ++ if (!data) ++ return -ENOMEM; ++ ++ for_each_online_cpu(cpu) { ++ unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; ++ ++ data[cpu] = x; ++ sum += x; ++ } ++ ++ len = sprintf(buf, "%lu", sum); ++ ++#ifdef CONFIG_SMP ++ for_each_online_cpu(cpu) { ++ if (data[cpu] && len < PAGE_SIZE - 20) ++ len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]); ++ } ++#endif ++ kfree(data); ++ return len + sprintf(buf + len, "\n"); ++} ++ ++static void clear_stat(struct kmem_cache *s, enum stat_item si) ++{ ++ int cpu; ++ ++ for_each_online_cpu(cpu) ++ per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; ++} ++ ++#define STAT_ATTR(si, text) \ ++static ssize_t text##_show(struct kmem_cache *s, char *buf) \ ++{ \ ++ return show_stat(s, buf, si); \ ++} \ ++static ssize_t text##_store(struct kmem_cache *s, \ ++ const char *buf, size_t length) \ ++{ \ ++ if (buf[0] != '0') \ ++ return -EINVAL; \ ++ clear_stat(s, si); \ ++ return length; \ ++} \ ++SLAB_ATTR(text); \ ++ ++STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); ++STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); ++STAT_ATTR(FREE_FASTPATH, free_fastpath); ++STAT_ATTR(FREE_SLOWPATH, free_slowpath); ++STAT_ATTR(FREE_FROZEN, free_frozen); ++STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); ++STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); ++STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); ++STAT_ATTR(ALLOC_SLAB, alloc_slab); ++STAT_ATTR(ALLOC_REFILL, alloc_refill); ++STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); ++STAT_ATTR(FREE_SLAB, free_slab); ++STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); ++STAT_ATTR(DEACTIVATE_FULL, deactivate_full); ++STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); ++STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); ++STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); ++STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); ++STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); ++STAT_ATTR(ORDER_FALLBACK, order_fallback); ++STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); ++STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); ++STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); ++STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); ++STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); ++STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); ++#endif ++ ++static struct attribute *slab_attrs[] = { ++ &slab_size_attr.attr, ++ &object_size_attr.attr, ++ &objs_per_slab_attr.attr, ++ &order_attr.attr, ++ &min_partial_attr.attr, ++ &cpu_partial_attr.attr, ++ &objects_attr.attr, ++ &objects_partial_attr.attr, ++ &partial_attr.attr, ++ &cpu_slabs_attr.attr, ++ &ctor_attr.attr, ++ &aliases_attr.attr, ++ &align_attr.attr, ++ &hwcache_align_attr.attr, ++ &reclaim_account_attr.attr, ++ &destroy_by_rcu_attr.attr, ++ &shrink_attr.attr, ++ &reserved_attr.attr, ++ &slabs_cpu_partial_attr.attr, ++#ifdef CONFIG_SLUB_DEBUG ++ &total_objects_attr.attr, ++ &slabs_attr.attr, ++ &sanity_checks_attr.attr, ++ &trace_attr.attr, ++ &red_zone_attr.attr, ++ &poison_attr.attr, ++ &store_user_attr.attr, ++ &validate_attr.attr, ++ &alloc_calls_attr.attr, ++ &free_calls_attr.attr, ++#endif ++#ifdef CONFIG_ZONE_DMA ++ &cache_dma_attr.attr, ++#endif ++#ifdef CONFIG_NUMA ++ &remote_node_defrag_ratio_attr.attr, ++#endif ++#ifdef CONFIG_SLUB_STATS ++ &alloc_fastpath_attr.attr, ++ &alloc_slowpath_attr.attr, ++ &free_fastpath_attr.attr, ++ &free_slowpath_attr.attr, ++ &free_frozen_attr.attr, ++ &free_add_partial_attr.attr, ++ &free_remove_partial_attr.attr, ++ &alloc_from_partial_attr.attr, ++ &alloc_slab_attr.attr, ++ &alloc_refill_attr.attr, ++ &alloc_node_mismatch_attr.attr, ++ &free_slab_attr.attr, ++ &cpuslab_flush_attr.attr, ++ &deactivate_full_attr.attr, ++ &deactivate_empty_attr.attr, ++ &deactivate_to_head_attr.attr, ++ &deactivate_to_tail_attr.attr, ++ &deactivate_remote_frees_attr.attr, ++ &deactivate_bypass_attr.attr, ++ &order_fallback_attr.attr, ++ &cmpxchg_double_fail_attr.attr, ++ &cmpxchg_double_cpu_fail_attr.attr, ++ &cpu_partial_alloc_attr.attr, ++ &cpu_partial_free_attr.attr, ++ &cpu_partial_node_attr.attr, ++ &cpu_partial_drain_attr.attr, ++#endif ++#ifdef CONFIG_FAILSLAB ++ &failslab_attr.attr, ++#endif ++ ++ NULL ++}; ++ ++static struct attribute_group slab_attr_group = { ++ .attrs = slab_attrs, ++}; ++ ++static ssize_t slab_attr_show(struct kobject *kobj, ++ struct attribute *attr, ++ char *buf) ++{ ++ struct slab_attribute *attribute; ++ struct kmem_cache *s; ++ int err; ++ ++ attribute = to_slab_attr(attr); ++ s = to_slab(kobj); ++ ++ if (!attribute->show) ++ return -EIO; ++ ++ err = attribute->show(s, buf); ++ ++ return err; ++} ++ ++static ssize_t slab_attr_store(struct kobject *kobj, ++ struct attribute *attr, ++ const char *buf, size_t len) ++{ ++ struct slab_attribute *attribute; ++ struct kmem_cache *s; ++ int err; ++ ++ attribute = to_slab_attr(attr); ++ s = to_slab(kobj); ++ ++ if (!attribute->store) ++ return -EIO; ++ ++ err = attribute->store(s, buf, len); ++#ifdef CONFIG_MEMCG_KMEM ++ if (slab_state >= FULL && err >= 0 && is_root_cache(s)) { ++ struct kmem_cache *c; ++ ++ mutex_lock(&slab_mutex); ++ if (s->max_attr_size < len) ++ s->max_attr_size = len; ++ ++ /* ++ * This is a best effort propagation, so this function's return ++ * value will be determined by the parent cache only. This is ++ * basically because not all attributes will have a well ++ * defined semantics for rollbacks - most of the actions will ++ * have permanent effects. ++ * ++ * Returning the error value of any of the children that fail ++ * is not 100 % defined, in the sense that users seeing the ++ * error code won't be able to know anything about the state of ++ * the cache. ++ * ++ * Only returning the error code for the parent cache at least ++ * has well defined semantics. The cache being written to ++ * directly either failed or succeeded, in which case we loop ++ * through the descendants with best-effort propagation. ++ */ ++ for_each_memcg_cache(c, s) ++ attribute->store(c, buf, len); ++ mutex_unlock(&slab_mutex); ++ } ++#endif ++ return err; ++} ++ ++static void memcg_propagate_slab_attrs(struct kmem_cache *s) ++{ ++#ifdef CONFIG_MEMCG_KMEM ++ int i; ++ char *buffer = NULL; ++ struct kmem_cache *root_cache; ++ ++ if (is_root_cache(s)) ++ return; ++ ++ root_cache = s->memcg_params.root_cache; ++ ++ /* ++ * This mean this cache had no attribute written. Therefore, no point ++ * in copying default values around ++ */ ++ if (!root_cache->max_attr_size) ++ return; ++ ++ for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) { ++ char mbuf[64]; ++ char *buf; ++ struct slab_attribute *attr = to_slab_attr(slab_attrs[i]); ++ ++ if (!attr || !attr->store || !attr->show) ++ continue; ++ ++ /* ++ * It is really bad that we have to allocate here, so we will ++ * do it only as a fallback. If we actually allocate, though, ++ * we can just use the allocated buffer until the end. ++ * ++ * Most of the slub attributes will tend to be very small in ++ * size, but sysfs allows buffers up to a page, so they can ++ * theoretically happen. ++ */ ++ if (buffer) ++ buf = buffer; ++ else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf)) ++ buf = mbuf; ++ else { ++ buffer = (char *) get_zeroed_page(GFP_KERNEL); ++ if (WARN_ON(!buffer)) ++ continue; ++ buf = buffer; ++ } ++ ++ attr->show(root_cache, buf); ++ attr->store(s, buf, strlen(buf)); ++ } ++ ++ if (buffer) ++ free_page((unsigned long)buffer); ++#endif ++} ++ ++static void kmem_cache_release(struct kobject *k) ++{ ++ slab_kmem_cache_release(to_slab(k)); ++} ++ ++static const struct sysfs_ops slab_sysfs_ops = { ++ .show = slab_attr_show, ++ .store = slab_attr_store, ++}; ++ ++static struct kobj_type slab_ktype = { ++ .sysfs_ops = &slab_sysfs_ops, ++ .release = kmem_cache_release, ++}; ++ ++static int uevent_filter(struct kset *kset, struct kobject *kobj) ++{ ++ struct kobj_type *ktype = get_ktype(kobj); ++ ++ if (ktype == &slab_ktype) ++ return 1; ++ return 0; ++} ++ ++static const struct kset_uevent_ops slab_uevent_ops = { ++ .filter = uevent_filter, ++}; ++ ++static struct kset *slab_kset; ++ ++static inline struct kset *cache_kset(struct kmem_cache *s) ++{ ++#ifdef CONFIG_MEMCG_KMEM ++ if (!is_root_cache(s)) ++ return s->memcg_params.root_cache->memcg_kset; ++#endif ++ return slab_kset; ++} ++ ++#define ID_STR_LENGTH 64 ++ ++/* Create a unique string id for a slab cache: ++ * ++ * Format :[flags-]size ++ */ ++static char *create_unique_id(struct kmem_cache *s) ++{ ++ char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); ++ char *p = name; ++ ++ BUG_ON(!name); ++ ++ *p++ = ':'; ++ /* ++ * First flags affecting slabcache operations. We will only ++ * get here for aliasable slabs so we do not need to support ++ * too many flags. The flags here must cover all flags that ++ * are matched during merging to guarantee that the id is ++ * unique. ++ */ ++ if (s->flags & SLAB_CACHE_DMA) ++ *p++ = 'd'; ++ if (s->flags & SLAB_RECLAIM_ACCOUNT) ++ *p++ = 'a'; ++ if (s->flags & SLAB_DEBUG_FREE) ++ *p++ = 'F'; ++ if (!(s->flags & SLAB_NOTRACK)) ++ *p++ = 't'; ++ if (p != name + 1) ++ *p++ = '-'; ++ p += sprintf(p, "%07d", s->size); ++ ++ BUG_ON(p > name + ID_STR_LENGTH - 1); ++ return name; ++} ++ ++static int sysfs_slab_add(struct kmem_cache *s) ++{ ++ int err; ++ const char *name; ++ int unmergeable = slab_unmergeable(s); ++ ++ if (unmergeable) { ++ /* ++ * Slabcache can never be merged so we can use the name proper. ++ * This is typically the case for debug situations. In that ++ * case we can catch duplicate names easily. ++ */ ++ sysfs_remove_link(&slab_kset->kobj, s->name); ++ name = s->name; ++ } else { ++ /* ++ * Create a unique name for the slab as a target ++ * for the symlinks. ++ */ ++ name = create_unique_id(s); ++ } ++ ++ s->kobj.kset = cache_kset(s); ++ err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); ++ if (err) ++ goto out_put_kobj; ++ ++ err = sysfs_create_group(&s->kobj, &slab_attr_group); ++ if (err) ++ goto out_del_kobj; ++ ++#ifdef CONFIG_MEMCG_KMEM ++ if (is_root_cache(s)) { ++ s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj); ++ if (!s->memcg_kset) { ++ err = -ENOMEM; ++ goto out_del_kobj; ++ } ++ } ++#endif ++ ++ kobject_uevent(&s->kobj, KOBJ_ADD); ++ if (!unmergeable) { ++ /* Setup first alias */ ++ sysfs_slab_alias(s, s->name); ++ } ++out: ++ if (!unmergeable) ++ kfree(name); ++ return err; ++out_del_kobj: ++ kobject_del(&s->kobj); ++out_put_kobj: ++ kobject_put(&s->kobj); ++ goto out; ++} ++ ++void sysfs_slab_remove(struct kmem_cache *s) ++{ ++ if (slab_state < FULL) ++ /* ++ * Sysfs has not been setup yet so no need to remove the ++ * cache from sysfs. ++ */ ++ return; ++ ++#ifdef CONFIG_MEMCG_KMEM ++ kset_unregister(s->memcg_kset); ++#endif ++ kobject_uevent(&s->kobj, KOBJ_REMOVE); ++ kobject_del(&s->kobj); ++ kobject_put(&s->kobj); ++} ++ ++/* ++ * Need to buffer aliases during bootup until sysfs becomes ++ * available lest we lose that information. ++ */ ++struct saved_alias { ++ struct kmem_cache *s; ++ const char *name; ++ struct saved_alias *next; ++}; ++ ++static struct saved_alias *alias_list; ++ ++static int sysfs_slab_alias(struct kmem_cache *s, const char *name) ++{ ++ struct saved_alias *al; ++ ++ if (slab_state == FULL) { ++ /* ++ * If we have a leftover link then remove it. ++ */ ++ sysfs_remove_link(&slab_kset->kobj, name); ++ return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); ++ } ++ ++ al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL); ++ if (!al) ++ return -ENOMEM; ++ ++ al->s = s; ++ al->name = name; ++ al->next = alias_list; ++ alias_list = al; ++ return 0; ++} ++ ++static int __init slab_sysfs_init(void) ++{ ++ struct kmem_cache *s; ++ int err; ++ ++ mutex_lock(&slab_mutex); ++ ++ slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj); ++ if (!slab_kset) { ++ mutex_unlock(&slab_mutex); ++ pr_err("Cannot register slab subsystem.\n"); ++ return -ENOSYS; ++ } ++ ++ slab_state = FULL; ++ ++ list_for_each_entry(s, &slab_caches, list) { ++ err = sysfs_slab_add(s); ++ if (err) ++ pr_err("SLUB: Unable to add boot slab %s to sysfs\n", ++ s->name); ++ } ++ ++ while (alias_list) { ++ struct saved_alias *al = alias_list; ++ ++ alias_list = alias_list->next; ++ err = sysfs_slab_alias(al->s, al->name); ++ if (err) ++ pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n", ++ al->name); ++ kfree(al); ++ } ++ ++ mutex_unlock(&slab_mutex); ++ resiliency_test(); ++ return 0; ++} ++ ++__initcall(slab_sysfs_init); ++#endif /* CONFIG_SYSFS */ ++ ++/* ++ * The /proc/slabinfo ABI ++ */ ++#ifdef CONFIG_SLABINFO ++void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) ++{ ++ unsigned long nr_slabs = 0; ++ unsigned long nr_objs = 0; ++ unsigned long nr_free = 0; ++ int node; ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) { ++ nr_slabs += node_nr_slabs(n); ++ nr_objs += node_nr_objs(n); ++ nr_free += count_partial(n, count_free); ++ } ++ ++ sinfo->active_objs = nr_objs - nr_free; ++ sinfo->num_objs = nr_objs; ++ sinfo->active_slabs = nr_slabs; ++ sinfo->num_slabs = nr_slabs; ++ sinfo->objects_per_slab = oo_objects(s->oo); ++ sinfo->cache_order = oo_order(s->oo); ++} ++ ++void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s) ++{ ++} ++ ++ssize_t slabinfo_write(struct file *file, const char __user *buffer, ++ size_t count, loff_t *ppos) ++{ ++ return -EIO; ++} ++#endif /* CONFIG_SLABINFO */ +diff -Nur linux-4.1.10.orig/mm/swap.c linux-4.1.10/mm/swap.c +--- linux-4.1.10.orig/mm/swap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/swap.c 2015-10-07 18:00:08.000000000 +0200 @@ -32,6 +32,7 @@ #include <linux/gfp.h> #include <linux/uio.h> @@ -25787,9 +62050,9 @@ diff -Nur linux-4.1.6.orig/mm/swap.c linux-4.1.6/mm/swap.c } static void lru_add_drain_per_cpu(struct work_struct *dummy) -diff -Nur linux-4.1.6.orig/mm/truncate.c linux-4.1.6/mm/truncate.c ---- linux-4.1.6.orig/mm/truncate.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/truncate.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/mm/truncate.c linux-4.1.10/mm/truncate.c +--- linux-4.1.10.orig/mm/truncate.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/truncate.c 2015-10-07 18:00:08.000000000 +0200 @@ -56,8 +56,11 @@ * protected by mapping->tree_lock. */ @@ -25804,9 +62067,9 @@ diff -Nur linux-4.1.6.orig/mm/truncate.c linux-4.1.6/mm/truncate.c __radix_tree_delete_node(&mapping->page_tree, node); unlock: spin_unlock_irq(&mapping->tree_lock); -diff -Nur linux-4.1.6.orig/mm/vmalloc.c linux-4.1.6/mm/vmalloc.c ---- linux-4.1.6.orig/mm/vmalloc.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/vmalloc.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/mm/vmalloc.c linux-4.1.10/mm/vmalloc.c +--- linux-4.1.10.orig/mm/vmalloc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/vmalloc.c 2015-10-07 18:00:08.000000000 +0200 @@ -819,7 +819,7 @@ struct vmap_block *vb; struct vmap_area *va; @@ -25858,9 +62121,9 @@ diff -Nur linux-4.1.6.orig/mm/vmalloc.c linux-4.1.6/mm/vmalloc.c rcu_read_unlock(); /* Allocate new block if nothing was found */ -diff -Nur linux-4.1.6.orig/mm/vmstat.c linux-4.1.6/mm/vmstat.c ---- linux-4.1.6.orig/mm/vmstat.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/vmstat.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/mm/vmstat.c linux-4.1.10/mm/vmstat.c +--- linux-4.1.10.orig/mm/vmstat.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/vmstat.c 2015-10-07 18:00:08.000000000 +0200 @@ -226,6 +226,7 @@ long x; long t; @@ -25909,9 +62172,9 @@ diff -Nur linux-4.1.6.orig/mm/vmstat.c linux-4.1.6/mm/vmstat.c } void __dec_zone_page_state(struct page *page, enum zone_stat_item item) -diff -Nur linux-4.1.6.orig/mm/workingset.c linux-4.1.6/mm/workingset.c ---- linux-4.1.6.orig/mm/workingset.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/workingset.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/mm/workingset.c linux-4.1.10/mm/workingset.c +--- linux-4.1.10.orig/mm/workingset.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/workingset.c 2015-10-07 18:00:08.000000000 +0200 @@ -264,7 +264,8 @@ * point where they would still be useful. */ @@ -25979,9 +62242,9 @@ diff -Nur linux-4.1.6.orig/mm/workingset.c linux-4.1.6/mm/workingset.c err: return ret; } -diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c ---- linux-4.1.6.orig/net/core/dev.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/core/dev.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/net/core/dev.c linux-4.1.10/net/core/dev.c +--- linux-4.1.10.orig/net/core/dev.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/core/dev.c 2015-10-07 18:00:08.000000000 +0200 @@ -184,6 +184,7 @@ static DEFINE_HASHTABLE(napi_hash, 8); @@ -26007,7 +62270,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c #endif } -@@ -856,7 +857,8 @@ +@@ -852,7 +853,8 @@ strcpy(name, dev->name); rcu_read_unlock(); if (read_seqcount_retry(&devnet_rename_seq, seq)) { @@ -26017,7 +62280,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c goto retry; } -@@ -1125,20 +1127,17 @@ +@@ -1121,20 +1123,17 @@ if (dev->flags & IFF_UP) return -EBUSY; @@ -26044,7 +62307,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c if (oldname[0] && !strchr(oldname, '%')) netdev_info(dev, "renamed from %s\n", oldname); -@@ -1151,11 +1150,12 @@ +@@ -1147,11 +1146,12 @@ if (ret) { memcpy(dev->name, oldname, IFNAMSIZ); dev->name_assign_type = old_assign_type; @@ -26060,7 +62323,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c netdev_adjacent_rename_links(dev, oldname); -@@ -1176,7 +1176,8 @@ +@@ -1172,7 +1172,8 @@ /* err >= 0 after dev_alloc_name() or stores the first errno */ if (err >= 0) { err = ret; @@ -26070,7 +62333,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c memcpy(dev->name, oldname, IFNAMSIZ); memcpy(oldname, newname, IFNAMSIZ); dev->name_assign_type = old_assign_type; -@@ -1189,6 +1190,11 @@ +@@ -1185,6 +1186,11 @@ } return err; @@ -26082,7 +62345,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c } /** -@@ -2218,6 +2224,7 @@ +@@ -2214,6 +2220,7 @@ sd->output_queue_tailp = &q->next_sched; raise_softirq_irqoff(NET_TX_SOFTIRQ); local_irq_restore(flags); @@ -26090,7 +62353,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c } void __netif_schedule(struct Qdisc *q) -@@ -2299,6 +2306,7 @@ +@@ -2295,6 +2302,7 @@ __this_cpu_write(softnet_data.completion_queue, skb); raise_softirq_irqoff(NET_TX_SOFTIRQ); local_irq_restore(flags); @@ -26098,7 +62361,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c } EXPORT_SYMBOL(__dev_kfree_skb_irq); -@@ -3366,6 +3374,7 @@ +@@ -3365,6 +3373,7 @@ rps_unlock(sd); local_irq_restore(flags); @@ -26106,7 +62369,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c atomic_long_inc(&skb->dev->rx_dropped); kfree_skb(skb); -@@ -3384,7 +3393,7 @@ +@@ -3383,7 +3392,7 @@ struct rps_dev_flow voidflow, *rflow = &voidflow; int cpu; @@ -26115,7 +62378,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c rcu_read_lock(); cpu = get_rps_cpu(skb->dev, skb, &rflow); -@@ -3394,13 +3403,13 @@ +@@ -3393,13 +3402,13 @@ ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); rcu_read_unlock(); @@ -26132,7 +62395,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c } return ret; } -@@ -3434,16 +3443,44 @@ +@@ -3433,16 +3442,44 @@ trace_netif_rx_ni_entry(skb); @@ -26181,7 +62444,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c static void net_tx_action(struct softirq_action *h) { struct softnet_data *sd = this_cpu_ptr(&softnet_data); -@@ -3485,7 +3522,7 @@ +@@ -3484,7 +3521,7 @@ head = head->next_sched; root_lock = qdisc_lock(q); @@ -26190,7 +62453,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c smp_mb__before_atomic(); clear_bit(__QDISC_STATE_SCHED, &q->state); -@@ -3884,7 +3921,7 @@ +@@ -3881,7 +3918,7 @@ skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { if (skb->dev == dev) { __skb_unlink(skb, &sd->input_pkt_queue); @@ -26199,7 +62462,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c input_queue_head_incr(sd); } } -@@ -3893,10 +3930,13 @@ +@@ -3890,10 +3927,13 @@ skb_queue_walk_safe(&sd->process_queue, skb, tmp) { if (skb->dev == dev) { __skb_unlink(skb, &sd->process_queue); @@ -26214,7 +62477,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c } static int napi_gro_complete(struct sk_buff *skb) -@@ -4347,6 +4387,7 @@ +@@ -4344,6 +4384,7 @@ sd->rps_ipi_list = NULL; local_irq_enable(); @@ -26222,7 +62485,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c /* Send pending IPI's to kick RPS processing on remote cpus. */ while (remsd) { -@@ -4360,6 +4401,7 @@ +@@ -4357,6 +4398,7 @@ } else #endif local_irq_enable(); @@ -26230,7 +62493,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c } static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) -@@ -4439,6 +4481,7 @@ +@@ -4438,6 +4480,7 @@ local_irq_save(flags); ____napi_schedule(this_cpu_ptr(&softnet_data), n); local_irq_restore(flags); @@ -26238,7 +62501,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c } EXPORT_SYMBOL(__napi_schedule); -@@ -7168,6 +7211,7 @@ +@@ -7167,6 +7210,7 @@ raise_softirq_irqoff(NET_TX_SOFTIRQ); local_irq_enable(); @@ -26246,7 +62509,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c /* Process offline CPU's input_pkt_queue */ while ((skb = __skb_dequeue(&oldsd->process_queue))) { -@@ -7178,6 +7222,9 @@ +@@ -7177,6 +7221,9 @@ netif_rx_ni(skb); input_queue_head_incr(oldsd); } @@ -26256,7 +62519,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c return NOTIFY_OK; } -@@ -7479,8 +7526,9 @@ +@@ -7478,8 +7525,9 @@ for_each_possible_cpu(i) { struct softnet_data *sd = &per_cpu(softnet_data, i); @@ -26268,9 +62531,7535 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c INIT_LIST_HEAD(&sd->poll_list); sd->output_queue_tailp = &sd->output_queue; #ifdef CONFIG_RPS -diff -Nur linux-4.1.6.orig/net/core/skbuff.c linux-4.1.6/net/core/skbuff.c ---- linux-4.1.6.orig/net/core/skbuff.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/core/skbuff.c 2015-09-08 23:49:08.445829687 +0200 +diff -Nur linux-4.1.10.orig/net/core/dev.c.orig linux-4.1.10/net/core/dev.c.orig +--- linux-4.1.10.orig/net/core/dev.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/net/core/dev.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,7522 @@ ++/* ++ * NET3 Protocol independent device support routines. ++ * ++ * This program is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License ++ * as published by the Free Software Foundation; either version ++ * 2 of the License, or (at your option) any later version. ++ * ++ * Derived from the non IP parts of dev.c 1.0.19 ++ * Authors: Ross Biro ++ * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> ++ * Mark Evans, <evansmp@uhura.aston.ac.uk> ++ * ++ * Additional Authors: ++ * Florian la Roche <rzsfl@rz.uni-sb.de> ++ * Alan Cox <gw4pts@gw4pts.ampr.org> ++ * David Hinds <dahinds@users.sourceforge.net> ++ * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> ++ * Adam Sulmicki <adam@cfar.umd.edu> ++ * Pekka Riikonen <priikone@poesidon.pspt.fi> ++ * ++ * Changes: ++ * D.J. Barrow : Fixed bug where dev->refcnt gets set ++ * to 2 if register_netdev gets called ++ * before net_dev_init & also removed a ++ * few lines of code in the process. ++ * Alan Cox : device private ioctl copies fields back. ++ * Alan Cox : Transmit queue code does relevant ++ * stunts to keep the queue safe. ++ * Alan Cox : Fixed double lock. ++ * Alan Cox : Fixed promisc NULL pointer trap ++ * ???????? : Support the full private ioctl range ++ * Alan Cox : Moved ioctl permission check into ++ * drivers ++ * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI ++ * Alan Cox : 100 backlog just doesn't cut it when ++ * you start doing multicast video 8) ++ * Alan Cox : Rewrote net_bh and list manager. ++ * Alan Cox : Fix ETH_P_ALL echoback lengths. ++ * Alan Cox : Took out transmit every packet pass ++ * Saved a few bytes in the ioctl handler ++ * Alan Cox : Network driver sets packet type before ++ * calling netif_rx. Saves a function ++ * call a packet. ++ * Alan Cox : Hashed net_bh() ++ * Richard Kooijman: Timestamp fixes. ++ * Alan Cox : Wrong field in SIOCGIFDSTADDR ++ * Alan Cox : Device lock protection. ++ * Alan Cox : Fixed nasty side effect of device close ++ * changes. ++ * Rudi Cilibrasi : Pass the right thing to ++ * set_mac_address() ++ * Dave Miller : 32bit quantity for the device lock to ++ * make it work out on a Sparc. ++ * Bjorn Ekwall : Added KERNELD hack. ++ * Alan Cox : Cleaned up the backlog initialise. ++ * Craig Metz : SIOCGIFCONF fix if space for under ++ * 1 device. ++ * Thomas Bogendoerfer : Return ENODEV for dev_open, if there ++ * is no device open function. ++ * Andi Kleen : Fix error reporting for SIOCGIFCONF ++ * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF ++ * Cyrus Durgin : Cleaned for KMOD ++ * Adam Sulmicki : Bug Fix : Network Device Unload ++ * A network device unload needs to purge ++ * the backlog queue. ++ * Paul Rusty Russell : SIOCSIFNAME ++ * Pekka Riikonen : Netdev boot-time settings code ++ * Andrew Morton : Make unregister_netdevice wait ++ * indefinitely on dev->refcnt ++ * J Hadi Salim : - Backlog queue sampling ++ * - netif_rx() feedback ++ */ ++ ++#include <asm/uaccess.h> ++#include <linux/bitops.h> ++#include <linux/capability.h> ++#include <linux/cpu.h> ++#include <linux/types.h> ++#include <linux/kernel.h> ++#include <linux/hash.h> ++#include <linux/slab.h> ++#include <linux/sched.h> ++#include <linux/mutex.h> ++#include <linux/string.h> ++#include <linux/mm.h> ++#include <linux/socket.h> ++#include <linux/sockios.h> ++#include <linux/errno.h> ++#include <linux/interrupt.h> ++#include <linux/if_ether.h> ++#include <linux/netdevice.h> ++#include <linux/etherdevice.h> ++#include <linux/ethtool.h> ++#include <linux/notifier.h> ++#include <linux/skbuff.h> ++#include <net/net_namespace.h> ++#include <net/sock.h> ++#include <linux/rtnetlink.h> ++#include <linux/stat.h> ++#include <net/dst.h> ++#include <net/pkt_sched.h> ++#include <net/checksum.h> ++#include <net/xfrm.h> ++#include <linux/highmem.h> ++#include <linux/init.h> ++#include <linux/module.h> ++#include <linux/netpoll.h> ++#include <linux/rcupdate.h> ++#include <linux/delay.h> ++#include <net/iw_handler.h> ++#include <asm/current.h> ++#include <linux/audit.h> ++#include <linux/dmaengine.h> ++#include <linux/err.h> ++#include <linux/ctype.h> ++#include <linux/if_arp.h> ++#include <linux/if_vlan.h> ++#include <linux/ip.h> ++#include <net/ip.h> ++#include <net/mpls.h> ++#include <linux/ipv6.h> ++#include <linux/in.h> ++#include <linux/jhash.h> ++#include <linux/random.h> ++#include <trace/events/napi.h> ++#include <trace/events/net.h> ++#include <trace/events/skb.h> ++#include <linux/pci.h> ++#include <linux/inetdevice.h> ++#include <linux/cpu_rmap.h> ++#include <linux/static_key.h> ++#include <linux/hashtable.h> ++#include <linux/vmalloc.h> ++#include <linux/if_macvlan.h> ++#include <linux/errqueue.h> ++#include <linux/hrtimer.h> ++ ++#include "net-sysfs.h" ++ ++/* Instead of increasing this, you should create a hash table. */ ++#define MAX_GRO_SKBS 8 ++ ++/* This should be increased if a protocol with a bigger head is added. */ ++#define GRO_MAX_HEAD (MAX_HEADER + 128) ++ ++static DEFINE_SPINLOCK(ptype_lock); ++static DEFINE_SPINLOCK(offload_lock); ++struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; ++struct list_head ptype_all __read_mostly; /* Taps */ ++static struct list_head offload_base __read_mostly; ++ ++static int netif_rx_internal(struct sk_buff *skb); ++static int call_netdevice_notifiers_info(unsigned long val, ++ struct net_device *dev, ++ struct netdev_notifier_info *info); ++ ++/* ++ * The @dev_base_head list is protected by @dev_base_lock and the rtnl ++ * semaphore. ++ * ++ * Pure readers hold dev_base_lock for reading, or rcu_read_lock() ++ * ++ * Writers must hold the rtnl semaphore while they loop through the ++ * dev_base_head list, and hold dev_base_lock for writing when they do the ++ * actual updates. This allows pure readers to access the list even ++ * while a writer is preparing to update it. ++ * ++ * To put it another way, dev_base_lock is held for writing only to ++ * protect against pure readers; the rtnl semaphore provides the ++ * protection against other writers. ++ * ++ * See, for example usages, register_netdevice() and ++ * unregister_netdevice(), which must be called with the rtnl ++ * semaphore held. ++ */ ++DEFINE_RWLOCK(dev_base_lock); ++EXPORT_SYMBOL(dev_base_lock); ++ ++/* protects napi_hash addition/deletion and napi_gen_id */ ++static DEFINE_SPINLOCK(napi_hash_lock); ++ ++static unsigned int napi_gen_id; ++static DEFINE_HASHTABLE(napi_hash, 8); ++ ++static seqcount_t devnet_rename_seq; ++ ++static inline void dev_base_seq_inc(struct net *net) ++{ ++ while (++net->dev_base_seq == 0); ++} ++ ++static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) ++{ ++ unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ)); ++ ++ return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; ++} ++ ++static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) ++{ ++ return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; ++} ++ ++static inline void rps_lock(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ spin_lock(&sd->input_pkt_queue.lock); ++#endif ++} ++ ++static inline void rps_unlock(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ spin_unlock(&sd->input_pkt_queue.lock); ++#endif ++} ++ ++/* Device list insertion */ ++static void list_netdevice(struct net_device *dev) ++{ ++ struct net *net = dev_net(dev); ++ ++ ASSERT_RTNL(); ++ ++ write_lock_bh(&dev_base_lock); ++ list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); ++ hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); ++ hlist_add_head_rcu(&dev->index_hlist, ++ dev_index_hash(net, dev->ifindex)); ++ write_unlock_bh(&dev_base_lock); ++ ++ dev_base_seq_inc(net); ++} ++ ++/* Device list removal ++ * caller must respect a RCU grace period before freeing/reusing dev ++ */ ++static void unlist_netdevice(struct net_device *dev) ++{ ++ ASSERT_RTNL(); ++ ++ /* Unlink dev from the device chain */ ++ write_lock_bh(&dev_base_lock); ++ list_del_rcu(&dev->dev_list); ++ hlist_del_rcu(&dev->name_hlist); ++ hlist_del_rcu(&dev->index_hlist); ++ write_unlock_bh(&dev_base_lock); ++ ++ dev_base_seq_inc(dev_net(dev)); ++} ++ ++/* ++ * Our notifier list ++ */ ++ ++static RAW_NOTIFIER_HEAD(netdev_chain); ++ ++/* ++ * Device drivers call our routines to queue packets here. We empty the ++ * queue in the local softnet handler. ++ */ ++ ++DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); ++EXPORT_PER_CPU_SYMBOL(softnet_data); ++ ++#ifdef CONFIG_LOCKDEP ++/* ++ * register_netdevice() inits txq->_xmit_lock and sets lockdep class ++ * according to dev->type ++ */ ++static const unsigned short netdev_lock_type[] = ++ {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, ++ ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, ++ ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, ++ ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, ++ ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, ++ ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, ++ ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, ++ ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, ++ ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, ++ ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, ++ ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, ++ ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, ++ ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, ++ ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, ++ ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; ++ ++static const char *const netdev_lock_name[] = ++ {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", ++ "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", ++ "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", ++ "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", ++ "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", ++ "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", ++ "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", ++ "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", ++ "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", ++ "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", ++ "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", ++ "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", ++ "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", ++ "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", ++ "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; ++ ++static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; ++static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; ++ ++static inline unsigned short netdev_lock_pos(unsigned short dev_type) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) ++ if (netdev_lock_type[i] == dev_type) ++ return i; ++ /* the last key is used by default */ ++ return ARRAY_SIZE(netdev_lock_type) - 1; ++} ++ ++static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, ++ unsigned short dev_type) ++{ ++ int i; ++ ++ i = netdev_lock_pos(dev_type); ++ lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], ++ netdev_lock_name[i]); ++} ++ ++static inline void netdev_set_addr_lockdep_class(struct net_device *dev) ++{ ++ int i; ++ ++ i = netdev_lock_pos(dev->type); ++ lockdep_set_class_and_name(&dev->addr_list_lock, ++ &netdev_addr_lock_key[i], ++ netdev_lock_name[i]); ++} ++#else ++static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, ++ unsigned short dev_type) ++{ ++} ++static inline void netdev_set_addr_lockdep_class(struct net_device *dev) ++{ ++} ++#endif ++ ++/******************************************************************************* ++ ++ Protocol management and registration routines ++ ++*******************************************************************************/ ++ ++/* ++ * Add a protocol ID to the list. Now that the input handler is ++ * smarter we can dispense with all the messy stuff that used to be ++ * here. ++ * ++ * BEWARE!!! Protocol handlers, mangling input packets, ++ * MUST BE last in hash buckets and checking protocol handlers ++ * MUST start from promiscuous ptype_all chain in net_bh. ++ * It is true now, do not change it. ++ * Explanation follows: if protocol handler, mangling packet, will ++ * be the first on list, it is not able to sense, that packet ++ * is cloned and should be copied-on-write, so that it will ++ * change it and subsequent readers will get broken packet. ++ * --ANK (980803) ++ */ ++ ++static inline struct list_head *ptype_head(const struct packet_type *pt) ++{ ++ if (pt->type == htons(ETH_P_ALL)) ++ return pt->dev ? &pt->dev->ptype_all : &ptype_all; ++ else ++ return pt->dev ? &pt->dev->ptype_specific : ++ &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; ++} ++ ++/** ++ * dev_add_pack - add packet handler ++ * @pt: packet type declaration ++ * ++ * Add a protocol handler to the networking stack. The passed &packet_type ++ * is linked into kernel lists and may not be freed until it has been ++ * removed from the kernel lists. ++ * ++ * This call does not sleep therefore it can not ++ * guarantee all CPU's that are in middle of receiving packets ++ * will see the new packet type (until the next received packet). ++ */ ++ ++void dev_add_pack(struct packet_type *pt) ++{ ++ struct list_head *head = ptype_head(pt); ++ ++ spin_lock(&ptype_lock); ++ list_add_rcu(&pt->list, head); ++ spin_unlock(&ptype_lock); ++} ++EXPORT_SYMBOL(dev_add_pack); ++ ++/** ++ * __dev_remove_pack - remove packet handler ++ * @pt: packet type declaration ++ * ++ * Remove a protocol handler that was previously added to the kernel ++ * protocol handlers by dev_add_pack(). The passed &packet_type is removed ++ * from the kernel lists and can be freed or reused once this function ++ * returns. ++ * ++ * The packet type might still be in use by receivers ++ * and must not be freed until after all the CPU's have gone ++ * through a quiescent state. ++ */ ++void __dev_remove_pack(struct packet_type *pt) ++{ ++ struct list_head *head = ptype_head(pt); ++ struct packet_type *pt1; ++ ++ spin_lock(&ptype_lock); ++ ++ list_for_each_entry(pt1, head, list) { ++ if (pt == pt1) { ++ list_del_rcu(&pt->list); ++ goto out; ++ } ++ } ++ ++ pr_warn("dev_remove_pack: %p not found\n", pt); ++out: ++ spin_unlock(&ptype_lock); ++} ++EXPORT_SYMBOL(__dev_remove_pack); ++ ++/** ++ * dev_remove_pack - remove packet handler ++ * @pt: packet type declaration ++ * ++ * Remove a protocol handler that was previously added to the kernel ++ * protocol handlers by dev_add_pack(). The passed &packet_type is removed ++ * from the kernel lists and can be freed or reused once this function ++ * returns. ++ * ++ * This call sleeps to guarantee that no CPU is looking at the packet ++ * type after return. ++ */ ++void dev_remove_pack(struct packet_type *pt) ++{ ++ __dev_remove_pack(pt); ++ ++ synchronize_net(); ++} ++EXPORT_SYMBOL(dev_remove_pack); ++ ++ ++/** ++ * dev_add_offload - register offload handlers ++ * @po: protocol offload declaration ++ * ++ * Add protocol offload handlers to the networking stack. The passed ++ * &proto_offload is linked into kernel lists and may not be freed until ++ * it has been removed from the kernel lists. ++ * ++ * This call does not sleep therefore it can not ++ * guarantee all CPU's that are in middle of receiving packets ++ * will see the new offload handlers (until the next received packet). ++ */ ++void dev_add_offload(struct packet_offload *po) ++{ ++ struct list_head *head = &offload_base; ++ ++ spin_lock(&offload_lock); ++ list_add_rcu(&po->list, head); ++ spin_unlock(&offload_lock); ++} ++EXPORT_SYMBOL(dev_add_offload); ++ ++/** ++ * __dev_remove_offload - remove offload handler ++ * @po: packet offload declaration ++ * ++ * Remove a protocol offload handler that was previously added to the ++ * kernel offload handlers by dev_add_offload(). The passed &offload_type ++ * is removed from the kernel lists and can be freed or reused once this ++ * function returns. ++ * ++ * The packet type might still be in use by receivers ++ * and must not be freed until after all the CPU's have gone ++ * through a quiescent state. ++ */ ++static void __dev_remove_offload(struct packet_offload *po) ++{ ++ struct list_head *head = &offload_base; ++ struct packet_offload *po1; ++ ++ spin_lock(&offload_lock); ++ ++ list_for_each_entry(po1, head, list) { ++ if (po == po1) { ++ list_del_rcu(&po->list); ++ goto out; ++ } ++ } ++ ++ pr_warn("dev_remove_offload: %p not found\n", po); ++out: ++ spin_unlock(&offload_lock); ++} ++ ++/** ++ * dev_remove_offload - remove packet offload handler ++ * @po: packet offload declaration ++ * ++ * Remove a packet offload handler that was previously added to the kernel ++ * offload handlers by dev_add_offload(). The passed &offload_type is ++ * removed from the kernel lists and can be freed or reused once this ++ * function returns. ++ * ++ * This call sleeps to guarantee that no CPU is looking at the packet ++ * type after return. ++ */ ++void dev_remove_offload(struct packet_offload *po) ++{ ++ __dev_remove_offload(po); ++ ++ synchronize_net(); ++} ++EXPORT_SYMBOL(dev_remove_offload); ++ ++/****************************************************************************** ++ ++ Device Boot-time Settings Routines ++ ++*******************************************************************************/ ++ ++/* Boot time configuration table */ ++static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; ++ ++/** ++ * netdev_boot_setup_add - add new setup entry ++ * @name: name of the device ++ * @map: configured settings for the device ++ * ++ * Adds new setup entry to the dev_boot_setup list. The function ++ * returns 0 on error and 1 on success. This is a generic routine to ++ * all netdevices. ++ */ ++static int netdev_boot_setup_add(char *name, struct ifmap *map) ++{ ++ struct netdev_boot_setup *s; ++ int i; ++ ++ s = dev_boot_setup; ++ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { ++ if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { ++ memset(s[i].name, 0, sizeof(s[i].name)); ++ strlcpy(s[i].name, name, IFNAMSIZ); ++ memcpy(&s[i].map, map, sizeof(s[i].map)); ++ break; ++ } ++ } ++ ++ return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; ++} ++ ++/** ++ * netdev_boot_setup_check - check boot time settings ++ * @dev: the netdevice ++ * ++ * Check boot time settings for the device. ++ * The found settings are set for the device to be used ++ * later in the device probing. ++ * Returns 0 if no settings found, 1 if they are. ++ */ ++int netdev_boot_setup_check(struct net_device *dev) ++{ ++ struct netdev_boot_setup *s = dev_boot_setup; ++ int i; ++ ++ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { ++ if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && ++ !strcmp(dev->name, s[i].name)) { ++ dev->irq = s[i].map.irq; ++ dev->base_addr = s[i].map.base_addr; ++ dev->mem_start = s[i].map.mem_start; ++ dev->mem_end = s[i].map.mem_end; ++ return 1; ++ } ++ } ++ return 0; ++} ++EXPORT_SYMBOL(netdev_boot_setup_check); ++ ++ ++/** ++ * netdev_boot_base - get address from boot time settings ++ * @prefix: prefix for network device ++ * @unit: id for network device ++ * ++ * Check boot time settings for the base address of device. ++ * The found settings are set for the device to be used ++ * later in the device probing. ++ * Returns 0 if no settings found. ++ */ ++unsigned long netdev_boot_base(const char *prefix, int unit) ++{ ++ const struct netdev_boot_setup *s = dev_boot_setup; ++ char name[IFNAMSIZ]; ++ int i; ++ ++ sprintf(name, "%s%d", prefix, unit); ++ ++ /* ++ * If device already registered then return base of 1 ++ * to indicate not to probe for this interface ++ */ ++ if (__dev_get_by_name(&init_net, name)) ++ return 1; ++ ++ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) ++ if (!strcmp(name, s[i].name)) ++ return s[i].map.base_addr; ++ return 0; ++} ++ ++/* ++ * Saves at boot time configured settings for any netdevice. ++ */ ++int __init netdev_boot_setup(char *str) ++{ ++ int ints[5]; ++ struct ifmap map; ++ ++ str = get_options(str, ARRAY_SIZE(ints), ints); ++ if (!str || !*str) ++ return 0; ++ ++ /* Save settings */ ++ memset(&map, 0, sizeof(map)); ++ if (ints[0] > 0) ++ map.irq = ints[1]; ++ if (ints[0] > 1) ++ map.base_addr = ints[2]; ++ if (ints[0] > 2) ++ map.mem_start = ints[3]; ++ if (ints[0] > 3) ++ map.mem_end = ints[4]; ++ ++ /* Add new entry to the list */ ++ return netdev_boot_setup_add(str, &map); ++} ++ ++__setup("netdev=", netdev_boot_setup); ++ ++/******************************************************************************* ++ ++ Device Interface Subroutines ++ ++*******************************************************************************/ ++ ++/** ++ * dev_get_iflink - get 'iflink' value of a interface ++ * @dev: targeted interface ++ * ++ * Indicates the ifindex the interface is linked to. ++ * Physical interfaces have the same 'ifindex' and 'iflink' values. ++ */ ++ ++int dev_get_iflink(const struct net_device *dev) ++{ ++ if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) ++ return dev->netdev_ops->ndo_get_iflink(dev); ++ ++ return dev->ifindex; ++} ++EXPORT_SYMBOL(dev_get_iflink); ++ ++/** ++ * __dev_get_by_name - find a device by its name ++ * @net: the applicable net namespace ++ * @name: name to find ++ * ++ * Find an interface by name. Must be called under RTNL semaphore ++ * or @dev_base_lock. If the name is found a pointer to the device ++ * is returned. If the name is not found then %NULL is returned. The ++ * reference counters are not incremented so the caller must be ++ * careful with locks. ++ */ ++ ++struct net_device *__dev_get_by_name(struct net *net, const char *name) ++{ ++ struct net_device *dev; ++ struct hlist_head *head = dev_name_hash(net, name); ++ ++ hlist_for_each_entry(dev, head, name_hlist) ++ if (!strncmp(dev->name, name, IFNAMSIZ)) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(__dev_get_by_name); ++ ++/** ++ * dev_get_by_name_rcu - find a device by its name ++ * @net: the applicable net namespace ++ * @name: name to find ++ * ++ * Find an interface by name. ++ * If the name is found a pointer to the device is returned. ++ * If the name is not found then %NULL is returned. ++ * The reference counters are not incremented so the caller must be ++ * careful with locks. The caller must hold RCU lock. ++ */ ++ ++struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) ++{ ++ struct net_device *dev; ++ struct hlist_head *head = dev_name_hash(net, name); ++ ++ hlist_for_each_entry_rcu(dev, head, name_hlist) ++ if (!strncmp(dev->name, name, IFNAMSIZ)) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(dev_get_by_name_rcu); ++ ++/** ++ * dev_get_by_name - find a device by its name ++ * @net: the applicable net namespace ++ * @name: name to find ++ * ++ * Find an interface by name. This can be called from any ++ * context and does its own locking. The returned handle has ++ * the usage count incremented and the caller must use dev_put() to ++ * release it when it is no longer needed. %NULL is returned if no ++ * matching device is found. ++ */ ++ ++struct net_device *dev_get_by_name(struct net *net, const char *name) ++{ ++ struct net_device *dev; ++ ++ rcu_read_lock(); ++ dev = dev_get_by_name_rcu(net, name); ++ if (dev) ++ dev_hold(dev); ++ rcu_read_unlock(); ++ return dev; ++} ++EXPORT_SYMBOL(dev_get_by_name); ++ ++/** ++ * __dev_get_by_index - find a device by its ifindex ++ * @net: the applicable net namespace ++ * @ifindex: index of device ++ * ++ * Search for an interface by index. Returns %NULL if the device ++ * is not found or a pointer to the device. The device has not ++ * had its reference counter increased so the caller must be careful ++ * about locking. The caller must hold either the RTNL semaphore ++ * or @dev_base_lock. ++ */ ++ ++struct net_device *__dev_get_by_index(struct net *net, int ifindex) ++{ ++ struct net_device *dev; ++ struct hlist_head *head = dev_index_hash(net, ifindex); ++ ++ hlist_for_each_entry(dev, head, index_hlist) ++ if (dev->ifindex == ifindex) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(__dev_get_by_index); ++ ++/** ++ * dev_get_by_index_rcu - find a device by its ifindex ++ * @net: the applicable net namespace ++ * @ifindex: index of device ++ * ++ * Search for an interface by index. Returns %NULL if the device ++ * is not found or a pointer to the device. The device has not ++ * had its reference counter increased so the caller must be careful ++ * about locking. The caller must hold RCU lock. ++ */ ++ ++struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) ++{ ++ struct net_device *dev; ++ struct hlist_head *head = dev_index_hash(net, ifindex); ++ ++ hlist_for_each_entry_rcu(dev, head, index_hlist) ++ if (dev->ifindex == ifindex) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(dev_get_by_index_rcu); ++ ++ ++/** ++ * dev_get_by_index - find a device by its ifindex ++ * @net: the applicable net namespace ++ * @ifindex: index of device ++ * ++ * Search for an interface by index. Returns NULL if the device ++ * is not found or a pointer to the device. The device returned has ++ * had a reference added and the pointer is safe until the user calls ++ * dev_put to indicate they have finished with it. ++ */ ++ ++struct net_device *dev_get_by_index(struct net *net, int ifindex) ++{ ++ struct net_device *dev; ++ ++ rcu_read_lock(); ++ dev = dev_get_by_index_rcu(net, ifindex); ++ if (dev) ++ dev_hold(dev); ++ rcu_read_unlock(); ++ return dev; ++} ++EXPORT_SYMBOL(dev_get_by_index); ++ ++/** ++ * netdev_get_name - get a netdevice name, knowing its ifindex. ++ * @net: network namespace ++ * @name: a pointer to the buffer where the name will be stored. ++ * @ifindex: the ifindex of the interface to get the name from. ++ * ++ * The use of raw_seqcount_begin() and cond_resched() before ++ * retrying is required as we want to give the writers a chance ++ * to complete when CONFIG_PREEMPT is not set. ++ */ ++int netdev_get_name(struct net *net, char *name, int ifindex) ++{ ++ struct net_device *dev; ++ unsigned int seq; ++ ++retry: ++ seq = raw_seqcount_begin(&devnet_rename_seq); ++ rcu_read_lock(); ++ dev = dev_get_by_index_rcu(net, ifindex); ++ if (!dev) { ++ rcu_read_unlock(); ++ return -ENODEV; ++ } ++ ++ strcpy(name, dev->name); ++ rcu_read_unlock(); ++ if (read_seqcount_retry(&devnet_rename_seq, seq)) { ++ cond_resched(); ++ goto retry; ++ } ++ ++ return 0; ++} ++ ++/** ++ * dev_getbyhwaddr_rcu - find a device by its hardware address ++ * @net: the applicable net namespace ++ * @type: media type of device ++ * @ha: hardware address ++ * ++ * Search for an interface by MAC address. Returns NULL if the device ++ * is not found or a pointer to the device. ++ * The caller must hold RCU or RTNL. ++ * The returned device has not had its ref count increased ++ * and the caller must therefore be careful about locking ++ * ++ */ ++ ++struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, ++ const char *ha) ++{ ++ struct net_device *dev; ++ ++ for_each_netdev_rcu(net, dev) ++ if (dev->type == type && ++ !memcmp(dev->dev_addr, ha, dev->addr_len)) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(dev_getbyhwaddr_rcu); ++ ++struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) ++{ ++ struct net_device *dev; ++ ++ ASSERT_RTNL(); ++ for_each_netdev(net, dev) ++ if (dev->type == type) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(__dev_getfirstbyhwtype); ++ ++struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) ++{ ++ struct net_device *dev, *ret = NULL; ++ ++ rcu_read_lock(); ++ for_each_netdev_rcu(net, dev) ++ if (dev->type == type) { ++ dev_hold(dev); ++ ret = dev; ++ break; ++ } ++ rcu_read_unlock(); ++ return ret; ++} ++EXPORT_SYMBOL(dev_getfirstbyhwtype); ++ ++/** ++ * __dev_get_by_flags - find any device with given flags ++ * @net: the applicable net namespace ++ * @if_flags: IFF_* values ++ * @mask: bitmask of bits in if_flags to check ++ * ++ * Search for any interface with the given flags. Returns NULL if a device ++ * is not found or a pointer to the device. Must be called inside ++ * rtnl_lock(), and result refcount is unchanged. ++ */ ++ ++struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, ++ unsigned short mask) ++{ ++ struct net_device *dev, *ret; ++ ++ ASSERT_RTNL(); ++ ++ ret = NULL; ++ for_each_netdev(net, dev) { ++ if (((dev->flags ^ if_flags) & mask) == 0) { ++ ret = dev; ++ break; ++ } ++ } ++ return ret; ++} ++EXPORT_SYMBOL(__dev_get_by_flags); ++ ++/** ++ * dev_valid_name - check if name is okay for network device ++ * @name: name string ++ * ++ * Network device names need to be valid file names to ++ * to allow sysfs to work. We also disallow any kind of ++ * whitespace. ++ */ ++bool dev_valid_name(const char *name) ++{ ++ if (*name == '\0') ++ return false; ++ if (strlen(name) >= IFNAMSIZ) ++ return false; ++ if (!strcmp(name, ".") || !strcmp(name, "..")) ++ return false; ++ ++ while (*name) { ++ if (*name == '/' || *name == ':' || isspace(*name)) ++ return false; ++ name++; ++ } ++ return true; ++} ++EXPORT_SYMBOL(dev_valid_name); ++ ++/** ++ * __dev_alloc_name - allocate a name for a device ++ * @net: network namespace to allocate the device name in ++ * @name: name format string ++ * @buf: scratch buffer and result name string ++ * ++ * Passed a format string - eg "lt%d" it will try and find a suitable ++ * id. It scans list of devices to build up a free map, then chooses ++ * the first empty slot. The caller must hold the dev_base or rtnl lock ++ * while allocating the name and adding the device in order to avoid ++ * duplicates. ++ * Limited to bits_per_byte * page size devices (ie 32K on most platforms). ++ * Returns the number of the unit assigned or a negative errno code. ++ */ ++ ++static int __dev_alloc_name(struct net *net, const char *name, char *buf) ++{ ++ int i = 0; ++ const char *p; ++ const int max_netdevices = 8*PAGE_SIZE; ++ unsigned long *inuse; ++ struct net_device *d; ++ ++ p = strnchr(name, IFNAMSIZ-1, '%'); ++ if (p) { ++ /* ++ * Verify the string as this thing may have come from ++ * the user. There must be either one "%d" and no other "%" ++ * characters. ++ */ ++ if (p[1] != 'd' || strchr(p + 2, '%')) ++ return -EINVAL; ++ ++ /* Use one page as a bit array of possible slots */ ++ inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); ++ if (!inuse) ++ return -ENOMEM; ++ ++ for_each_netdev(net, d) { ++ if (!sscanf(d->name, name, &i)) ++ continue; ++ if (i < 0 || i >= max_netdevices) ++ continue; ++ ++ /* avoid cases where sscanf is not exact inverse of printf */ ++ snprintf(buf, IFNAMSIZ, name, i); ++ if (!strncmp(buf, d->name, IFNAMSIZ)) ++ set_bit(i, inuse); ++ } ++ ++ i = find_first_zero_bit(inuse, max_netdevices); ++ free_page((unsigned long) inuse); ++ } ++ ++ if (buf != name) ++ snprintf(buf, IFNAMSIZ, name, i); ++ if (!__dev_get_by_name(net, buf)) ++ return i; ++ ++ /* It is possible to run out of possible slots ++ * when the name is long and there isn't enough space left ++ * for the digits, or if all bits are used. ++ */ ++ return -ENFILE; ++} ++ ++/** ++ * dev_alloc_name - allocate a name for a device ++ * @dev: device ++ * @name: name format string ++ * ++ * Passed a format string - eg "lt%d" it will try and find a suitable ++ * id. It scans list of devices to build up a free map, then chooses ++ * the first empty slot. The caller must hold the dev_base or rtnl lock ++ * while allocating the name and adding the device in order to avoid ++ * duplicates. ++ * Limited to bits_per_byte * page size devices (ie 32K on most platforms). ++ * Returns the number of the unit assigned or a negative errno code. ++ */ ++ ++int dev_alloc_name(struct net_device *dev, const char *name) ++{ ++ char buf[IFNAMSIZ]; ++ struct net *net; ++ int ret; ++ ++ BUG_ON(!dev_net(dev)); ++ net = dev_net(dev); ++ ret = __dev_alloc_name(net, name, buf); ++ if (ret >= 0) ++ strlcpy(dev->name, buf, IFNAMSIZ); ++ return ret; ++} ++EXPORT_SYMBOL(dev_alloc_name); ++ ++static int dev_alloc_name_ns(struct net *net, ++ struct net_device *dev, ++ const char *name) ++{ ++ char buf[IFNAMSIZ]; ++ int ret; ++ ++ ret = __dev_alloc_name(net, name, buf); ++ if (ret >= 0) ++ strlcpy(dev->name, buf, IFNAMSIZ); ++ return ret; ++} ++ ++static int dev_get_valid_name(struct net *net, ++ struct net_device *dev, ++ const char *name) ++{ ++ BUG_ON(!net); ++ ++ if (!dev_valid_name(name)) ++ return -EINVAL; ++ ++ if (strchr(name, '%')) ++ return dev_alloc_name_ns(net, dev, name); ++ else if (__dev_get_by_name(net, name)) ++ return -EEXIST; ++ else if (dev->name != name) ++ strlcpy(dev->name, name, IFNAMSIZ); ++ ++ return 0; ++} ++ ++/** ++ * dev_change_name - change name of a device ++ * @dev: device ++ * @newname: name (or format string) must be at least IFNAMSIZ ++ * ++ * Change name of a device, can pass format strings "eth%d". ++ * for wildcarding. ++ */ ++int dev_change_name(struct net_device *dev, const char *newname) ++{ ++ unsigned char old_assign_type; ++ char oldname[IFNAMSIZ]; ++ int err = 0; ++ int ret; ++ struct net *net; ++ ++ ASSERT_RTNL(); ++ BUG_ON(!dev_net(dev)); ++ ++ net = dev_net(dev); ++ if (dev->flags & IFF_UP) ++ return -EBUSY; ++ ++ write_seqcount_begin(&devnet_rename_seq); ++ ++ if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { ++ write_seqcount_end(&devnet_rename_seq); ++ return 0; ++ } ++ ++ memcpy(oldname, dev->name, IFNAMSIZ); ++ ++ err = dev_get_valid_name(net, dev, newname); ++ if (err < 0) { ++ write_seqcount_end(&devnet_rename_seq); ++ return err; ++ } ++ ++ if (oldname[0] && !strchr(oldname, '%')) ++ netdev_info(dev, "renamed from %s\n", oldname); ++ ++ old_assign_type = dev->name_assign_type; ++ dev->name_assign_type = NET_NAME_RENAMED; ++ ++rollback: ++ ret = device_rename(&dev->dev, dev->name); ++ if (ret) { ++ memcpy(dev->name, oldname, IFNAMSIZ); ++ dev->name_assign_type = old_assign_type; ++ write_seqcount_end(&devnet_rename_seq); ++ return ret; ++ } ++ ++ write_seqcount_end(&devnet_rename_seq); ++ ++ netdev_adjacent_rename_links(dev, oldname); ++ ++ write_lock_bh(&dev_base_lock); ++ hlist_del_rcu(&dev->name_hlist); ++ write_unlock_bh(&dev_base_lock); ++ ++ synchronize_rcu(); ++ ++ write_lock_bh(&dev_base_lock); ++ hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); ++ write_unlock_bh(&dev_base_lock); ++ ++ ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); ++ ret = notifier_to_errno(ret); ++ ++ if (ret) { ++ /* err >= 0 after dev_alloc_name() or stores the first errno */ ++ if (err >= 0) { ++ err = ret; ++ write_seqcount_begin(&devnet_rename_seq); ++ memcpy(dev->name, oldname, IFNAMSIZ); ++ memcpy(oldname, newname, IFNAMSIZ); ++ dev->name_assign_type = old_assign_type; ++ old_assign_type = NET_NAME_RENAMED; ++ goto rollback; ++ } else { ++ pr_err("%s: name change rollback failed: %d\n", ++ dev->name, ret); ++ } ++ } ++ ++ return err; ++} ++ ++/** ++ * dev_set_alias - change ifalias of a device ++ * @dev: device ++ * @alias: name up to IFALIASZ ++ * @len: limit of bytes to copy from info ++ * ++ * Set ifalias for a device, ++ */ ++int dev_set_alias(struct net_device *dev, const char *alias, size_t len) ++{ ++ char *new_ifalias; ++ ++ ASSERT_RTNL(); ++ ++ if (len >= IFALIASZ) ++ return -EINVAL; ++ ++ if (!len) { ++ kfree(dev->ifalias); ++ dev->ifalias = NULL; ++ return 0; ++ } ++ ++ new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL); ++ if (!new_ifalias) ++ return -ENOMEM; ++ dev->ifalias = new_ifalias; ++ ++ strlcpy(dev->ifalias, alias, len+1); ++ return len; ++} ++ ++ ++/** ++ * netdev_features_change - device changes features ++ * @dev: device to cause notification ++ * ++ * Called to indicate a device has changed features. ++ */ ++void netdev_features_change(struct net_device *dev) ++{ ++ call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); ++} ++EXPORT_SYMBOL(netdev_features_change); ++ ++/** ++ * netdev_state_change - device changes state ++ * @dev: device to cause notification ++ * ++ * Called to indicate a device has changed state. This function calls ++ * the notifier chains for netdev_chain and sends a NEWLINK message ++ * to the routing socket. ++ */ ++void netdev_state_change(struct net_device *dev) ++{ ++ if (dev->flags & IFF_UP) { ++ struct netdev_notifier_change_info change_info; ++ ++ change_info.flags_changed = 0; ++ call_netdevice_notifiers_info(NETDEV_CHANGE, dev, ++ &change_info.info); ++ rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); ++ } ++} ++EXPORT_SYMBOL(netdev_state_change); ++ ++/** ++ * netdev_notify_peers - notify network peers about existence of @dev ++ * @dev: network device ++ * ++ * Generate traffic such that interested network peers are aware of ++ * @dev, such as by generating a gratuitous ARP. This may be used when ++ * a device wants to inform the rest of the network about some sort of ++ * reconfiguration such as a failover event or virtual machine ++ * migration. ++ */ ++void netdev_notify_peers(struct net_device *dev) ++{ ++ rtnl_lock(); ++ call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); ++ rtnl_unlock(); ++} ++EXPORT_SYMBOL(netdev_notify_peers); ++ ++static int __dev_open(struct net_device *dev) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ int ret; ++ ++ ASSERT_RTNL(); ++ ++ if (!netif_device_present(dev)) ++ return -ENODEV; ++ ++ /* Block netpoll from trying to do any rx path servicing. ++ * If we don't do this there is a chance ndo_poll_controller ++ * or ndo_poll may be running while we open the device ++ */ ++ netpoll_poll_disable(dev); ++ ++ ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); ++ ret = notifier_to_errno(ret); ++ if (ret) ++ return ret; ++ ++ set_bit(__LINK_STATE_START, &dev->state); ++ ++ if (ops->ndo_validate_addr) ++ ret = ops->ndo_validate_addr(dev); ++ ++ if (!ret && ops->ndo_open) ++ ret = ops->ndo_open(dev); ++ ++ netpoll_poll_enable(dev); ++ ++ if (ret) ++ clear_bit(__LINK_STATE_START, &dev->state); ++ else { ++ dev->flags |= IFF_UP; ++ dev_set_rx_mode(dev); ++ dev_activate(dev); ++ add_device_randomness(dev->dev_addr, dev->addr_len); ++ } ++ ++ return ret; ++} ++ ++/** ++ * dev_open - prepare an interface for use. ++ * @dev: device to open ++ * ++ * Takes a device from down to up state. The device's private open ++ * function is invoked and then the multicast lists are loaded. Finally ++ * the device is moved into the up state and a %NETDEV_UP message is ++ * sent to the netdev notifier chain. ++ * ++ * Calling this function on an active interface is a nop. On a failure ++ * a negative errno code is returned. ++ */ ++int dev_open(struct net_device *dev) ++{ ++ int ret; ++ ++ if (dev->flags & IFF_UP) ++ return 0; ++ ++ ret = __dev_open(dev); ++ if (ret < 0) ++ return ret; ++ ++ rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); ++ call_netdevice_notifiers(NETDEV_UP, dev); ++ ++ return ret; ++} ++EXPORT_SYMBOL(dev_open); ++ ++static int __dev_close_many(struct list_head *head) ++{ ++ struct net_device *dev; ++ ++ ASSERT_RTNL(); ++ might_sleep(); ++ ++ list_for_each_entry(dev, head, close_list) { ++ /* Temporarily disable netpoll until the interface is down */ ++ netpoll_poll_disable(dev); ++ ++ call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); ++ ++ clear_bit(__LINK_STATE_START, &dev->state); ++ ++ /* Synchronize to scheduled poll. We cannot touch poll list, it ++ * can be even on different cpu. So just clear netif_running(). ++ * ++ * dev->stop() will invoke napi_disable() on all of it's ++ * napi_struct instances on this device. ++ */ ++ smp_mb__after_atomic(); /* Commit netif_running(). */ ++ } ++ ++ dev_deactivate_many(head); ++ ++ list_for_each_entry(dev, head, close_list) { ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ /* ++ * Call the device specific close. This cannot fail. ++ * Only if device is UP ++ * ++ * We allow it to be called even after a DETACH hot-plug ++ * event. ++ */ ++ if (ops->ndo_stop) ++ ops->ndo_stop(dev); ++ ++ dev->flags &= ~IFF_UP; ++ netpoll_poll_enable(dev); ++ } ++ ++ return 0; ++} ++ ++static int __dev_close(struct net_device *dev) ++{ ++ int retval; ++ LIST_HEAD(single); ++ ++ list_add(&dev->close_list, &single); ++ retval = __dev_close_many(&single); ++ list_del(&single); ++ ++ return retval; ++} ++ ++int dev_close_many(struct list_head *head, bool unlink) ++{ ++ struct net_device *dev, *tmp; ++ ++ /* Remove the devices that don't need to be closed */ ++ list_for_each_entry_safe(dev, tmp, head, close_list) ++ if (!(dev->flags & IFF_UP)) ++ list_del_init(&dev->close_list); ++ ++ __dev_close_many(head); ++ ++ list_for_each_entry_safe(dev, tmp, head, close_list) { ++ rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); ++ call_netdevice_notifiers(NETDEV_DOWN, dev); ++ if (unlink) ++ list_del_init(&dev->close_list); ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(dev_close_many); ++ ++/** ++ * dev_close - shutdown an interface. ++ * @dev: device to shutdown ++ * ++ * This function moves an active device into down state. A ++ * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device ++ * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier ++ * chain. ++ */ ++int dev_close(struct net_device *dev) ++{ ++ if (dev->flags & IFF_UP) { ++ LIST_HEAD(single); ++ ++ list_add(&dev->close_list, &single); ++ dev_close_many(&single, true); ++ list_del(&single); ++ } ++ return 0; ++} ++EXPORT_SYMBOL(dev_close); ++ ++ ++/** ++ * dev_disable_lro - disable Large Receive Offload on a device ++ * @dev: device ++ * ++ * Disable Large Receive Offload (LRO) on a net device. Must be ++ * called under RTNL. This is needed if received packets may be ++ * forwarded to another interface. ++ */ ++void dev_disable_lro(struct net_device *dev) ++{ ++ struct net_device *lower_dev; ++ struct list_head *iter; ++ ++ dev->wanted_features &= ~NETIF_F_LRO; ++ netdev_update_features(dev); ++ ++ if (unlikely(dev->features & NETIF_F_LRO)) ++ netdev_WARN(dev, "failed to disable LRO!\n"); ++ ++ netdev_for_each_lower_dev(dev, lower_dev, iter) ++ dev_disable_lro(lower_dev); ++} ++EXPORT_SYMBOL(dev_disable_lro); ++ ++static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, ++ struct net_device *dev) ++{ ++ struct netdev_notifier_info info; ++ ++ netdev_notifier_info_init(&info, dev); ++ return nb->notifier_call(nb, val, &info); ++} ++ ++static int dev_boot_phase = 1; ++ ++/** ++ * register_netdevice_notifier - register a network notifier block ++ * @nb: notifier ++ * ++ * Register a notifier to be called when network device events occur. ++ * The notifier passed is linked into the kernel structures and must ++ * not be reused until it has been unregistered. A negative errno code ++ * is returned on a failure. ++ * ++ * When registered all registration and up events are replayed ++ * to the new notifier to allow device to have a race free ++ * view of the network device list. ++ */ ++ ++int register_netdevice_notifier(struct notifier_block *nb) ++{ ++ struct net_device *dev; ++ struct net_device *last; ++ struct net *net; ++ int err; ++ ++ rtnl_lock(); ++ err = raw_notifier_chain_register(&netdev_chain, nb); ++ if (err) ++ goto unlock; ++ if (dev_boot_phase) ++ goto unlock; ++ for_each_net(net) { ++ for_each_netdev(net, dev) { ++ err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); ++ err = notifier_to_errno(err); ++ if (err) ++ goto rollback; ++ ++ if (!(dev->flags & IFF_UP)) ++ continue; ++ ++ call_netdevice_notifier(nb, NETDEV_UP, dev); ++ } ++ } ++ ++unlock: ++ rtnl_unlock(); ++ return err; ++ ++rollback: ++ last = dev; ++ for_each_net(net) { ++ for_each_netdev(net, dev) { ++ if (dev == last) ++ goto outroll; ++ ++ if (dev->flags & IFF_UP) { ++ call_netdevice_notifier(nb, NETDEV_GOING_DOWN, ++ dev); ++ call_netdevice_notifier(nb, NETDEV_DOWN, dev); ++ } ++ call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); ++ } ++ } ++ ++outroll: ++ raw_notifier_chain_unregister(&netdev_chain, nb); ++ goto unlock; ++} ++EXPORT_SYMBOL(register_netdevice_notifier); ++ ++/** ++ * unregister_netdevice_notifier - unregister a network notifier block ++ * @nb: notifier ++ * ++ * Unregister a notifier previously registered by ++ * register_netdevice_notifier(). The notifier is unlinked into the ++ * kernel structures and may then be reused. A negative errno code ++ * is returned on a failure. ++ * ++ * After unregistering unregister and down device events are synthesized ++ * for all devices on the device list to the removed notifier to remove ++ * the need for special case cleanup code. ++ */ ++ ++int unregister_netdevice_notifier(struct notifier_block *nb) ++{ ++ struct net_device *dev; ++ struct net *net; ++ int err; ++ ++ rtnl_lock(); ++ err = raw_notifier_chain_unregister(&netdev_chain, nb); ++ if (err) ++ goto unlock; ++ ++ for_each_net(net) { ++ for_each_netdev(net, dev) { ++ if (dev->flags & IFF_UP) { ++ call_netdevice_notifier(nb, NETDEV_GOING_DOWN, ++ dev); ++ call_netdevice_notifier(nb, NETDEV_DOWN, dev); ++ } ++ call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); ++ } ++ } ++unlock: ++ rtnl_unlock(); ++ return err; ++} ++EXPORT_SYMBOL(unregister_netdevice_notifier); ++ ++/** ++ * call_netdevice_notifiers_info - call all network notifier blocks ++ * @val: value passed unmodified to notifier function ++ * @dev: net_device pointer passed unmodified to notifier function ++ * @info: notifier information data ++ * ++ * Call all network notifier blocks. Parameters and return value ++ * are as for raw_notifier_call_chain(). ++ */ ++ ++static int call_netdevice_notifiers_info(unsigned long val, ++ struct net_device *dev, ++ struct netdev_notifier_info *info) ++{ ++ ASSERT_RTNL(); ++ netdev_notifier_info_init(info, dev); ++ return raw_notifier_call_chain(&netdev_chain, val, info); ++} ++ ++/** ++ * call_netdevice_notifiers - call all network notifier blocks ++ * @val: value passed unmodified to notifier function ++ * @dev: net_device pointer passed unmodified to notifier function ++ * ++ * Call all network notifier blocks. Parameters and return value ++ * are as for raw_notifier_call_chain(). ++ */ ++ ++int call_netdevice_notifiers(unsigned long val, struct net_device *dev) ++{ ++ struct netdev_notifier_info info; ++ ++ return call_netdevice_notifiers_info(val, dev, &info); ++} ++EXPORT_SYMBOL(call_netdevice_notifiers); ++ ++#ifdef CONFIG_NET_CLS_ACT ++static struct static_key ingress_needed __read_mostly; ++ ++void net_inc_ingress_queue(void) ++{ ++ static_key_slow_inc(&ingress_needed); ++} ++EXPORT_SYMBOL_GPL(net_inc_ingress_queue); ++ ++void net_dec_ingress_queue(void) ++{ ++ static_key_slow_dec(&ingress_needed); ++} ++EXPORT_SYMBOL_GPL(net_dec_ingress_queue); ++#endif ++ ++static struct static_key netstamp_needed __read_mostly; ++#ifdef HAVE_JUMP_LABEL ++/* We are not allowed to call static_key_slow_dec() from irq context ++ * If net_disable_timestamp() is called from irq context, defer the ++ * static_key_slow_dec() calls. ++ */ ++static atomic_t netstamp_needed_deferred; ++#endif ++ ++void net_enable_timestamp(void) ++{ ++#ifdef HAVE_JUMP_LABEL ++ int deferred = atomic_xchg(&netstamp_needed_deferred, 0); ++ ++ if (deferred) { ++ while (--deferred) ++ static_key_slow_dec(&netstamp_needed); ++ return; ++ } ++#endif ++ static_key_slow_inc(&netstamp_needed); ++} ++EXPORT_SYMBOL(net_enable_timestamp); ++ ++void net_disable_timestamp(void) ++{ ++#ifdef HAVE_JUMP_LABEL ++ if (in_interrupt()) { ++ atomic_inc(&netstamp_needed_deferred); ++ return; ++ } ++#endif ++ static_key_slow_dec(&netstamp_needed); ++} ++EXPORT_SYMBOL(net_disable_timestamp); ++ ++static inline void net_timestamp_set(struct sk_buff *skb) ++{ ++ skb->tstamp.tv64 = 0; ++ if (static_key_false(&netstamp_needed)) ++ __net_timestamp(skb); ++} ++ ++#define net_timestamp_check(COND, SKB) \ ++ if (static_key_false(&netstamp_needed)) { \ ++ if ((COND) && !(SKB)->tstamp.tv64) \ ++ __net_timestamp(SKB); \ ++ } \ ++ ++bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb) ++{ ++ unsigned int len; ++ ++ if (!(dev->flags & IFF_UP)) ++ return false; ++ ++ len = dev->mtu + dev->hard_header_len + VLAN_HLEN; ++ if (skb->len <= len) ++ return true; ++ ++ /* if TSO is enabled, we don't care about the length as the packet ++ * could be forwarded without being segmented before ++ */ ++ if (skb_is_gso(skb)) ++ return true; ++ ++ return false; ++} ++EXPORT_SYMBOL_GPL(is_skb_forwardable); ++ ++int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) ++{ ++ if (skb_orphan_frags(skb, GFP_ATOMIC) || ++ unlikely(!is_skb_forwardable(dev, skb))) { ++ atomic_long_inc(&dev->rx_dropped); ++ kfree_skb(skb); ++ return NET_RX_DROP; ++ } ++ ++ skb_scrub_packet(skb, true); ++ skb->priority = 0; ++ skb->protocol = eth_type_trans(skb, dev); ++ skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(__dev_forward_skb); ++ ++/** ++ * dev_forward_skb - loopback an skb to another netif ++ * ++ * @dev: destination network device ++ * @skb: buffer to forward ++ * ++ * return values: ++ * NET_RX_SUCCESS (no congestion) ++ * NET_RX_DROP (packet was dropped, but freed) ++ * ++ * dev_forward_skb can be used for injecting an skb from the ++ * start_xmit function of one device into the receive queue ++ * of another device. ++ * ++ * The receiving device may be in another namespace, so ++ * we have to clear all information in the skb that could ++ * impact namespace isolation. ++ */ ++int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) ++{ ++ return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); ++} ++EXPORT_SYMBOL_GPL(dev_forward_skb); ++ ++static inline int deliver_skb(struct sk_buff *skb, ++ struct packet_type *pt_prev, ++ struct net_device *orig_dev) ++{ ++ if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) ++ return -ENOMEM; ++ atomic_inc(&skb->users); ++ return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); ++} ++ ++static inline void deliver_ptype_list_skb(struct sk_buff *skb, ++ struct packet_type **pt, ++ struct net_device *orig_dev, ++ __be16 type, ++ struct list_head *ptype_list) ++{ ++ struct packet_type *ptype, *pt_prev = *pt; ++ ++ list_for_each_entry_rcu(ptype, ptype_list, list) { ++ if (ptype->type != type) ++ continue; ++ if (pt_prev) ++ deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = ptype; ++ } ++ *pt = pt_prev; ++} ++ ++static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) ++{ ++ if (!ptype->af_packet_priv || !skb->sk) ++ return false; ++ ++ if (ptype->id_match) ++ return ptype->id_match(ptype, skb->sk); ++ else if ((struct sock *)ptype->af_packet_priv == skb->sk) ++ return true; ++ ++ return false; ++} ++ ++/* ++ * Support routine. Sends outgoing frames to any network ++ * taps currently in use. ++ */ ++ ++static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) ++{ ++ struct packet_type *ptype; ++ struct sk_buff *skb2 = NULL; ++ struct packet_type *pt_prev = NULL; ++ struct list_head *ptype_list = &ptype_all; ++ ++ rcu_read_lock(); ++again: ++ list_for_each_entry_rcu(ptype, ptype_list, list) { ++ /* Never send packets back to the socket ++ * they originated from - MvS (miquels@drinkel.ow.org) ++ */ ++ if (skb_loop_sk(ptype, skb)) ++ continue; ++ ++ if (pt_prev) { ++ deliver_skb(skb2, pt_prev, skb->dev); ++ pt_prev = ptype; ++ continue; ++ } ++ ++ /* need to clone skb, done only once */ ++ skb2 = skb_clone(skb, GFP_ATOMIC); ++ if (!skb2) ++ goto out_unlock; ++ ++ net_timestamp_set(skb2); ++ ++ /* skb->nh should be correctly ++ * set by sender, so that the second statement is ++ * just protection against buggy protocols. ++ */ ++ skb_reset_mac_header(skb2); ++ ++ if (skb_network_header(skb2) < skb2->data || ++ skb_network_header(skb2) > skb_tail_pointer(skb2)) { ++ net_crit_ratelimited("protocol %04x is buggy, dev %s\n", ++ ntohs(skb2->protocol), ++ dev->name); ++ skb_reset_network_header(skb2); ++ } ++ ++ skb2->transport_header = skb2->network_header; ++ skb2->pkt_type = PACKET_OUTGOING; ++ pt_prev = ptype; ++ } ++ ++ if (ptype_list == &ptype_all) { ++ ptype_list = &dev->ptype_all; ++ goto again; ++ } ++out_unlock: ++ if (pt_prev) ++ pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); ++ rcu_read_unlock(); ++} ++ ++/** ++ * netif_setup_tc - Handle tc mappings on real_num_tx_queues change ++ * @dev: Network device ++ * @txq: number of queues available ++ * ++ * If real_num_tx_queues is changed the tc mappings may no longer be ++ * valid. To resolve this verify the tc mapping remains valid and if ++ * not NULL the mapping. With no priorities mapping to this ++ * offset/count pair it will no longer be used. In the worst case TC0 ++ * is invalid nothing can be done so disable priority mappings. If is ++ * expected that drivers will fix this mapping if they can before ++ * calling netif_set_real_num_tx_queues. ++ */ ++static void netif_setup_tc(struct net_device *dev, unsigned int txq) ++{ ++ int i; ++ struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; ++ ++ /* If TC0 is invalidated disable TC mapping */ ++ if (tc->offset + tc->count > txq) { ++ pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); ++ dev->num_tc = 0; ++ return; ++ } ++ ++ /* Invalidated prio to tc mappings set to TC0 */ ++ for (i = 1; i < TC_BITMASK + 1; i++) { ++ int q = netdev_get_prio_tc_map(dev, i); ++ ++ tc = &dev->tc_to_txq[q]; ++ if (tc->offset + tc->count > txq) { ++ pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", ++ i, q); ++ netdev_set_prio_tc_map(dev, i, 0); ++ } ++ } ++} ++ ++#ifdef CONFIG_XPS ++static DEFINE_MUTEX(xps_map_mutex); ++#define xmap_dereference(P) \ ++ rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) ++ ++static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps, ++ int cpu, u16 index) ++{ ++ struct xps_map *map = NULL; ++ int pos; ++ ++ if (dev_maps) ++ map = xmap_dereference(dev_maps->cpu_map[cpu]); ++ ++ for (pos = 0; map && pos < map->len; pos++) { ++ if (map->queues[pos] == index) { ++ if (map->len > 1) { ++ map->queues[pos] = map->queues[--map->len]; ++ } else { ++ RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL); ++ kfree_rcu(map, rcu); ++ map = NULL; ++ } ++ break; ++ } ++ } ++ ++ return map; ++} ++ ++static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) ++{ ++ struct xps_dev_maps *dev_maps; ++ int cpu, i; ++ bool active = false; ++ ++ mutex_lock(&xps_map_mutex); ++ dev_maps = xmap_dereference(dev->xps_maps); ++ ++ if (!dev_maps) ++ goto out_no_maps; ++ ++ for_each_possible_cpu(cpu) { ++ for (i = index; i < dev->num_tx_queues; i++) { ++ if (!remove_xps_queue(dev_maps, cpu, i)) ++ break; ++ } ++ if (i == dev->num_tx_queues) ++ active = true; ++ } ++ ++ if (!active) { ++ RCU_INIT_POINTER(dev->xps_maps, NULL); ++ kfree_rcu(dev_maps, rcu); ++ } ++ ++ for (i = index; i < dev->num_tx_queues; i++) ++ netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), ++ NUMA_NO_NODE); ++ ++out_no_maps: ++ mutex_unlock(&xps_map_mutex); ++} ++ ++static struct xps_map *expand_xps_map(struct xps_map *map, ++ int cpu, u16 index) ++{ ++ struct xps_map *new_map; ++ int alloc_len = XPS_MIN_MAP_ALLOC; ++ int i, pos; ++ ++ for (pos = 0; map && pos < map->len; pos++) { ++ if (map->queues[pos] != index) ++ continue; ++ return map; ++ } ++ ++ /* Need to add queue to this CPU's existing map */ ++ if (map) { ++ if (pos < map->alloc_len) ++ return map; ++ ++ alloc_len = map->alloc_len * 2; ++ } ++ ++ /* Need to allocate new map to store queue on this CPU's map */ ++ new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, ++ cpu_to_node(cpu)); ++ if (!new_map) ++ return NULL; ++ ++ for (i = 0; i < pos; i++) ++ new_map->queues[i] = map->queues[i]; ++ new_map->alloc_len = alloc_len; ++ new_map->len = pos; ++ ++ return new_map; ++} ++ ++int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, ++ u16 index) ++{ ++ struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; ++ struct xps_map *map, *new_map; ++ int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES); ++ int cpu, numa_node_id = -2; ++ bool active = false; ++ ++ mutex_lock(&xps_map_mutex); ++ ++ dev_maps = xmap_dereference(dev->xps_maps); ++ ++ /* allocate memory for queue storage */ ++ for_each_online_cpu(cpu) { ++ if (!cpumask_test_cpu(cpu, mask)) ++ continue; ++ ++ if (!new_dev_maps) ++ new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); ++ if (!new_dev_maps) { ++ mutex_unlock(&xps_map_mutex); ++ return -ENOMEM; ++ } ++ ++ map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : ++ NULL; ++ ++ map = expand_xps_map(map, cpu, index); ++ if (!map) ++ goto error; ++ ++ RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); ++ } ++ ++ if (!new_dev_maps) ++ goto out_no_new_maps; ++ ++ for_each_possible_cpu(cpu) { ++ if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) { ++ /* add queue to CPU maps */ ++ int pos = 0; ++ ++ map = xmap_dereference(new_dev_maps->cpu_map[cpu]); ++ while ((pos < map->len) && (map->queues[pos] != index)) ++ pos++; ++ ++ if (pos == map->len) ++ map->queues[map->len++] = index; ++#ifdef CONFIG_NUMA ++ if (numa_node_id == -2) ++ numa_node_id = cpu_to_node(cpu); ++ else if (numa_node_id != cpu_to_node(cpu)) ++ numa_node_id = -1; ++#endif ++ } else if (dev_maps) { ++ /* fill in the new device map from the old device map */ ++ map = xmap_dereference(dev_maps->cpu_map[cpu]); ++ RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); ++ } ++ ++ } ++ ++ rcu_assign_pointer(dev->xps_maps, new_dev_maps); ++ ++ /* Cleanup old maps */ ++ if (dev_maps) { ++ for_each_possible_cpu(cpu) { ++ new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); ++ map = xmap_dereference(dev_maps->cpu_map[cpu]); ++ if (map && map != new_map) ++ kfree_rcu(map, rcu); ++ } ++ ++ kfree_rcu(dev_maps, rcu); ++ } ++ ++ dev_maps = new_dev_maps; ++ active = true; ++ ++out_no_new_maps: ++ /* update Tx queue numa node */ ++ netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), ++ (numa_node_id >= 0) ? numa_node_id : ++ NUMA_NO_NODE); ++ ++ if (!dev_maps) ++ goto out_no_maps; ++ ++ /* removes queue from unused CPUs */ ++ for_each_possible_cpu(cpu) { ++ if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) ++ continue; ++ ++ if (remove_xps_queue(dev_maps, cpu, index)) ++ active = true; ++ } ++ ++ /* free map if not active */ ++ if (!active) { ++ RCU_INIT_POINTER(dev->xps_maps, NULL); ++ kfree_rcu(dev_maps, rcu); ++ } ++ ++out_no_maps: ++ mutex_unlock(&xps_map_mutex); ++ ++ return 0; ++error: ++ /* remove any maps that we added */ ++ for_each_possible_cpu(cpu) { ++ new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); ++ map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : ++ NULL; ++ if (new_map && new_map != map) ++ kfree(new_map); ++ } ++ ++ mutex_unlock(&xps_map_mutex); ++ ++ kfree(new_dev_maps); ++ return -ENOMEM; ++} ++EXPORT_SYMBOL(netif_set_xps_queue); ++ ++#endif ++/* ++ * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues ++ * greater then real_num_tx_queues stale skbs on the qdisc must be flushed. ++ */ ++int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) ++{ ++ int rc; ++ ++ if (txq < 1 || txq > dev->num_tx_queues) ++ return -EINVAL; ++ ++ if (dev->reg_state == NETREG_REGISTERED || ++ dev->reg_state == NETREG_UNREGISTERING) { ++ ASSERT_RTNL(); ++ ++ rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, ++ txq); ++ if (rc) ++ return rc; ++ ++ if (dev->num_tc) ++ netif_setup_tc(dev, txq); ++ ++ if (txq < dev->real_num_tx_queues) { ++ qdisc_reset_all_tx_gt(dev, txq); ++#ifdef CONFIG_XPS ++ netif_reset_xps_queues_gt(dev, txq); ++#endif ++ } ++ } ++ ++ dev->real_num_tx_queues = txq; ++ return 0; ++} ++EXPORT_SYMBOL(netif_set_real_num_tx_queues); ++ ++#ifdef CONFIG_SYSFS ++/** ++ * netif_set_real_num_rx_queues - set actual number of RX queues used ++ * @dev: Network device ++ * @rxq: Actual number of RX queues ++ * ++ * This must be called either with the rtnl_lock held or before ++ * registration of the net device. Returns 0 on success, or a ++ * negative error code. If called before registration, it always ++ * succeeds. ++ */ ++int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) ++{ ++ int rc; ++ ++ if (rxq < 1 || rxq > dev->num_rx_queues) ++ return -EINVAL; ++ ++ if (dev->reg_state == NETREG_REGISTERED) { ++ ASSERT_RTNL(); ++ ++ rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, ++ rxq); ++ if (rc) ++ return rc; ++ } ++ ++ dev->real_num_rx_queues = rxq; ++ return 0; ++} ++EXPORT_SYMBOL(netif_set_real_num_rx_queues); ++#endif ++ ++/** ++ * netif_get_num_default_rss_queues - default number of RSS queues ++ * ++ * This routine should set an upper limit on the number of RSS queues ++ * used by default by multiqueue devices. ++ */ ++int netif_get_num_default_rss_queues(void) ++{ ++ return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); ++} ++EXPORT_SYMBOL(netif_get_num_default_rss_queues); ++ ++static inline void __netif_reschedule(struct Qdisc *q) ++{ ++ struct softnet_data *sd; ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ sd = this_cpu_ptr(&softnet_data); ++ q->next_sched = NULL; ++ *sd->output_queue_tailp = q; ++ sd->output_queue_tailp = &q->next_sched; ++ raise_softirq_irqoff(NET_TX_SOFTIRQ); ++ local_irq_restore(flags); ++} ++ ++void __netif_schedule(struct Qdisc *q) ++{ ++ if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) ++ __netif_reschedule(q); ++} ++EXPORT_SYMBOL(__netif_schedule); ++ ++struct dev_kfree_skb_cb { ++ enum skb_free_reason reason; ++}; ++ ++static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) ++{ ++ return (struct dev_kfree_skb_cb *)skb->cb; ++} ++ ++void netif_schedule_queue(struct netdev_queue *txq) ++{ ++ rcu_read_lock(); ++ if (!(txq->state & QUEUE_STATE_ANY_XOFF)) { ++ struct Qdisc *q = rcu_dereference(txq->qdisc); ++ ++ __netif_schedule(q); ++ } ++ rcu_read_unlock(); ++} ++EXPORT_SYMBOL(netif_schedule_queue); ++ ++/** ++ * netif_wake_subqueue - allow sending packets on subqueue ++ * @dev: network device ++ * @queue_index: sub queue index ++ * ++ * Resume individual transmit queue of a device with multiple transmit queues. ++ */ ++void netif_wake_subqueue(struct net_device *dev, u16 queue_index) ++{ ++ struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); ++ ++ if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) { ++ struct Qdisc *q; ++ ++ rcu_read_lock(); ++ q = rcu_dereference(txq->qdisc); ++ __netif_schedule(q); ++ rcu_read_unlock(); ++ } ++} ++EXPORT_SYMBOL(netif_wake_subqueue); ++ ++void netif_tx_wake_queue(struct netdev_queue *dev_queue) ++{ ++ if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { ++ struct Qdisc *q; ++ ++ rcu_read_lock(); ++ q = rcu_dereference(dev_queue->qdisc); ++ __netif_schedule(q); ++ rcu_read_unlock(); ++ } ++} ++EXPORT_SYMBOL(netif_tx_wake_queue); ++ ++void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) ++{ ++ unsigned long flags; ++ ++ if (likely(atomic_read(&skb->users) == 1)) { ++ smp_rmb(); ++ atomic_set(&skb->users, 0); ++ } else if (likely(!atomic_dec_and_test(&skb->users))) { ++ return; ++ } ++ get_kfree_skb_cb(skb)->reason = reason; ++ local_irq_save(flags); ++ skb->next = __this_cpu_read(softnet_data.completion_queue); ++ __this_cpu_write(softnet_data.completion_queue, skb); ++ raise_softirq_irqoff(NET_TX_SOFTIRQ); ++ local_irq_restore(flags); ++} ++EXPORT_SYMBOL(__dev_kfree_skb_irq); ++ ++void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) ++{ ++ if (in_irq() || irqs_disabled()) ++ __dev_kfree_skb_irq(skb, reason); ++ else ++ dev_kfree_skb(skb); ++} ++EXPORT_SYMBOL(__dev_kfree_skb_any); ++ ++ ++/** ++ * netif_device_detach - mark device as removed ++ * @dev: network device ++ * ++ * Mark device as removed from system and therefore no longer available. ++ */ ++void netif_device_detach(struct net_device *dev) ++{ ++ if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && ++ netif_running(dev)) { ++ netif_tx_stop_all_queues(dev); ++ } ++} ++EXPORT_SYMBOL(netif_device_detach); ++ ++/** ++ * netif_device_attach - mark device as attached ++ * @dev: network device ++ * ++ * Mark device as attached from system and restart if needed. ++ */ ++void netif_device_attach(struct net_device *dev) ++{ ++ if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && ++ netif_running(dev)) { ++ netif_tx_wake_all_queues(dev); ++ __netdev_watchdog_up(dev); ++ } ++} ++EXPORT_SYMBOL(netif_device_attach); ++ ++static void skb_warn_bad_offload(const struct sk_buff *skb) ++{ ++ static const netdev_features_t null_features = 0; ++ struct net_device *dev = skb->dev; ++ const char *driver = ""; ++ ++ if (!net_ratelimit()) ++ return; ++ ++ if (dev && dev->dev.parent) ++ driver = dev_driver_string(dev->dev.parent); ++ ++ WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d " ++ "gso_type=%d ip_summed=%d\n", ++ driver, dev ? &dev->features : &null_features, ++ skb->sk ? &skb->sk->sk_route_caps : &null_features, ++ skb->len, skb->data_len, skb_shinfo(skb)->gso_size, ++ skb_shinfo(skb)->gso_type, skb->ip_summed); ++} ++ ++/* ++ * Invalidate hardware checksum when packet is to be mangled, and ++ * complete checksum manually on outgoing path. ++ */ ++int skb_checksum_help(struct sk_buff *skb) ++{ ++ __wsum csum; ++ int ret = 0, offset; ++ ++ if (skb->ip_summed == CHECKSUM_COMPLETE) ++ goto out_set_summed; ++ ++ if (unlikely(skb_shinfo(skb)->gso_size)) { ++ skb_warn_bad_offload(skb); ++ return -EINVAL; ++ } ++ ++ /* Before computing a checksum, we should make sure no frag could ++ * be modified by an external entity : checksum could be wrong. ++ */ ++ if (skb_has_shared_frag(skb)) { ++ ret = __skb_linearize(skb); ++ if (ret) ++ goto out; ++ } ++ ++ offset = skb_checksum_start_offset(skb); ++ BUG_ON(offset >= skb_headlen(skb)); ++ csum = skb_checksum(skb, offset, skb->len - offset, 0); ++ ++ offset += skb->csum_offset; ++ BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); ++ ++ if (skb_cloned(skb) && ++ !skb_clone_writable(skb, offset + sizeof(__sum16))) { ++ ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); ++ if (ret) ++ goto out; ++ } ++ ++ *(__sum16 *)(skb->data + offset) = csum_fold(csum); ++out_set_summed: ++ skb->ip_summed = CHECKSUM_NONE; ++out: ++ return ret; ++} ++EXPORT_SYMBOL(skb_checksum_help); ++ ++__be16 skb_network_protocol(struct sk_buff *skb, int *depth) ++{ ++ __be16 type = skb->protocol; ++ ++ /* Tunnel gso handlers can set protocol to ethernet. */ ++ if (type == htons(ETH_P_TEB)) { ++ struct ethhdr *eth; ++ ++ if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) ++ return 0; ++ ++ eth = (struct ethhdr *)skb_mac_header(skb); ++ type = eth->h_proto; ++ } ++ ++ return __vlan_get_protocol(skb, type, depth); ++} ++ ++/** ++ * skb_mac_gso_segment - mac layer segmentation handler. ++ * @skb: buffer to segment ++ * @features: features for the output path (see dev->features) ++ */ ++struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, ++ netdev_features_t features) ++{ ++ struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); ++ struct packet_offload *ptype; ++ int vlan_depth = skb->mac_len; ++ __be16 type = skb_network_protocol(skb, &vlan_depth); ++ ++ if (unlikely(!type)) ++ return ERR_PTR(-EINVAL); ++ ++ __skb_pull(skb, vlan_depth); ++ ++ rcu_read_lock(); ++ list_for_each_entry_rcu(ptype, &offload_base, list) { ++ if (ptype->type == type && ptype->callbacks.gso_segment) { ++ segs = ptype->callbacks.gso_segment(skb, features); ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ ++ __skb_push(skb, skb->data - skb_mac_header(skb)); ++ ++ return segs; ++} ++EXPORT_SYMBOL(skb_mac_gso_segment); ++ ++ ++/* openvswitch calls this on rx path, so we need a different check. ++ */ ++static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) ++{ ++ if (tx_path) ++ return skb->ip_summed != CHECKSUM_PARTIAL; ++ else ++ return skb->ip_summed == CHECKSUM_NONE; ++} ++ ++/** ++ * __skb_gso_segment - Perform segmentation on skb. ++ * @skb: buffer to segment ++ * @features: features for the output path (see dev->features) ++ * @tx_path: whether it is called in TX path ++ * ++ * This function segments the given skb and returns a list of segments. ++ * ++ * It may return NULL if the skb requires no segmentation. This is ++ * only possible when GSO is used for verifying header integrity. ++ */ ++struct sk_buff *__skb_gso_segment(struct sk_buff *skb, ++ netdev_features_t features, bool tx_path) ++{ ++ if (unlikely(skb_needs_check(skb, tx_path))) { ++ int err; ++ ++ skb_warn_bad_offload(skb); ++ ++ err = skb_cow_head(skb, 0); ++ if (err < 0) ++ return ERR_PTR(err); ++ } ++ ++ SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); ++ SKB_GSO_CB(skb)->encap_level = 0; ++ ++ skb_reset_mac_header(skb); ++ skb_reset_mac_len(skb); ++ ++ return skb_mac_gso_segment(skb, features); ++} ++EXPORT_SYMBOL(__skb_gso_segment); ++ ++/* Take action when hardware reception checksum errors are detected. */ ++#ifdef CONFIG_BUG ++void netdev_rx_csum_fault(struct net_device *dev) ++{ ++ if (net_ratelimit()) { ++ pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>"); ++ dump_stack(); ++ } ++} ++EXPORT_SYMBOL(netdev_rx_csum_fault); ++#endif ++ ++/* Actually, we should eliminate this check as soon as we know, that: ++ * 1. IOMMU is present and allows to map all the memory. ++ * 2. No high memory really exists on this machine. ++ */ ++ ++static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) ++{ ++#ifdef CONFIG_HIGHMEM ++ int i; ++ if (!(dev->features & NETIF_F_HIGHDMA)) { ++ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { ++ skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; ++ if (PageHighMem(skb_frag_page(frag))) ++ return 1; ++ } ++ } ++ ++ if (PCI_DMA_BUS_IS_PHYS) { ++ struct device *pdev = dev->dev.parent; ++ ++ if (!pdev) ++ return 0; ++ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { ++ skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; ++ dma_addr_t addr = page_to_phys(skb_frag_page(frag)); ++ if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) ++ return 1; ++ } ++ } ++#endif ++ return 0; ++} ++ ++/* If MPLS offload request, verify we are testing hardware MPLS features ++ * instead of standard features for the netdev. ++ */ ++#if IS_ENABLED(CONFIG_NET_MPLS_GSO) ++static netdev_features_t net_mpls_features(struct sk_buff *skb, ++ netdev_features_t features, ++ __be16 type) ++{ ++ if (eth_p_mpls(type)) ++ features &= skb->dev->mpls_features; ++ ++ return features; ++} ++#else ++static netdev_features_t net_mpls_features(struct sk_buff *skb, ++ netdev_features_t features, ++ __be16 type) ++{ ++ return features; ++} ++#endif ++ ++static netdev_features_t harmonize_features(struct sk_buff *skb, ++ netdev_features_t features) ++{ ++ int tmp; ++ __be16 type; ++ ++ type = skb_network_protocol(skb, &tmp); ++ features = net_mpls_features(skb, features, type); ++ ++ if (skb->ip_summed != CHECKSUM_NONE && ++ !can_checksum_protocol(features, type)) { ++ features &= ~NETIF_F_ALL_CSUM; ++ } else if (illegal_highdma(skb->dev, skb)) { ++ features &= ~NETIF_F_SG; ++ } ++ ++ return features; ++} ++ ++netdev_features_t passthru_features_check(struct sk_buff *skb, ++ struct net_device *dev, ++ netdev_features_t features) ++{ ++ return features; ++} ++EXPORT_SYMBOL(passthru_features_check); ++ ++static netdev_features_t dflt_features_check(const struct sk_buff *skb, ++ struct net_device *dev, ++ netdev_features_t features) ++{ ++ return vlan_features_check(skb, features); ++} ++ ++netdev_features_t netif_skb_features(struct sk_buff *skb) ++{ ++ struct net_device *dev = skb->dev; ++ netdev_features_t features = dev->features; ++ u16 gso_segs = skb_shinfo(skb)->gso_segs; ++ ++ if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs) ++ features &= ~NETIF_F_GSO_MASK; ++ ++ /* If encapsulation offload request, verify we are testing ++ * hardware encapsulation features instead of standard ++ * features for the netdev ++ */ ++ if (skb->encapsulation) ++ features &= dev->hw_enc_features; ++ ++ if (skb_vlan_tagged(skb)) ++ features = netdev_intersect_features(features, ++ dev->vlan_features | ++ NETIF_F_HW_VLAN_CTAG_TX | ++ NETIF_F_HW_VLAN_STAG_TX); ++ ++ if (dev->netdev_ops->ndo_features_check) ++ features &= dev->netdev_ops->ndo_features_check(skb, dev, ++ features); ++ else ++ features &= dflt_features_check(skb, dev, features); ++ ++ return harmonize_features(skb, features); ++} ++EXPORT_SYMBOL(netif_skb_features); ++ ++static int xmit_one(struct sk_buff *skb, struct net_device *dev, ++ struct netdev_queue *txq, bool more) ++{ ++ unsigned int len; ++ int rc; ++ ++ if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all)) ++ dev_queue_xmit_nit(skb, dev); ++ ++ len = skb->len; ++ trace_net_dev_start_xmit(skb, dev); ++ rc = netdev_start_xmit(skb, dev, txq, more); ++ trace_net_dev_xmit(skb, rc, dev, len); ++ ++ return rc; ++} ++ ++struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, ++ struct netdev_queue *txq, int *ret) ++{ ++ struct sk_buff *skb = first; ++ int rc = NETDEV_TX_OK; ++ ++ while (skb) { ++ struct sk_buff *next = skb->next; ++ ++ skb->next = NULL; ++ rc = xmit_one(skb, dev, txq, next != NULL); ++ if (unlikely(!dev_xmit_complete(rc))) { ++ skb->next = next; ++ goto out; ++ } ++ ++ skb = next; ++ if (netif_xmit_stopped(txq) && skb) { ++ rc = NETDEV_TX_BUSY; ++ break; ++ } ++ } ++ ++out: ++ *ret = rc; ++ return skb; ++} ++ ++static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, ++ netdev_features_t features) ++{ ++ if (skb_vlan_tag_present(skb) && ++ !vlan_hw_offload_capable(features, skb->vlan_proto)) ++ skb = __vlan_hwaccel_push_inside(skb); ++ return skb; ++} ++ ++static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev) ++{ ++ netdev_features_t features; ++ ++ if (skb->next) ++ return skb; ++ ++ features = netif_skb_features(skb); ++ skb = validate_xmit_vlan(skb, features); ++ if (unlikely(!skb)) ++ goto out_null; ++ ++ if (netif_needs_gso(skb, features)) { ++ struct sk_buff *segs; ++ ++ segs = skb_gso_segment(skb, features); ++ if (IS_ERR(segs)) { ++ goto out_kfree_skb; ++ } else if (segs) { ++ consume_skb(skb); ++ skb = segs; ++ } ++ } else { ++ if (skb_needs_linearize(skb, features) && ++ __skb_linearize(skb)) ++ goto out_kfree_skb; ++ ++ /* If packet is not checksummed and device does not ++ * support checksumming for this protocol, complete ++ * checksumming here. ++ */ ++ if (skb->ip_summed == CHECKSUM_PARTIAL) { ++ if (skb->encapsulation) ++ skb_set_inner_transport_header(skb, ++ skb_checksum_start_offset(skb)); ++ else ++ skb_set_transport_header(skb, ++ skb_checksum_start_offset(skb)); ++ if (!(features & NETIF_F_ALL_CSUM) && ++ skb_checksum_help(skb)) ++ goto out_kfree_skb; ++ } ++ } ++ ++ return skb; ++ ++out_kfree_skb: ++ kfree_skb(skb); ++out_null: ++ return NULL; ++} ++ ++struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev) ++{ ++ struct sk_buff *next, *head = NULL, *tail; ++ ++ for (; skb != NULL; skb = next) { ++ next = skb->next; ++ skb->next = NULL; ++ ++ /* in case skb wont be segmented, point to itself */ ++ skb->prev = skb; ++ ++ skb = validate_xmit_skb(skb, dev); ++ if (!skb) ++ continue; ++ ++ if (!head) ++ head = skb; ++ else ++ tail->next = skb; ++ /* If skb was segmented, skb->prev points to ++ * the last segment. If not, it still contains skb. ++ */ ++ tail = skb->prev; ++ } ++ return head; ++} ++ ++static void qdisc_pkt_len_init(struct sk_buff *skb) ++{ ++ const struct skb_shared_info *shinfo = skb_shinfo(skb); ++ ++ qdisc_skb_cb(skb)->pkt_len = skb->len; ++ ++ /* To get more precise estimation of bytes sent on wire, ++ * we add to pkt_len the headers size of all segments ++ */ ++ if (shinfo->gso_size) { ++ unsigned int hdr_len; ++ u16 gso_segs = shinfo->gso_segs; ++ ++ /* mac layer + network layer */ ++ hdr_len = skb_transport_header(skb) - skb_mac_header(skb); ++ ++ /* + transport layer */ ++ if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) ++ hdr_len += tcp_hdrlen(skb); ++ else ++ hdr_len += sizeof(struct udphdr); ++ ++ if (shinfo->gso_type & SKB_GSO_DODGY) ++ gso_segs = DIV_ROUND_UP(skb->len - hdr_len, ++ shinfo->gso_size); ++ ++ qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; ++ } ++} ++ ++static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, ++ struct net_device *dev, ++ struct netdev_queue *txq) ++{ ++ spinlock_t *root_lock = qdisc_lock(q); ++ bool contended; ++ int rc; ++ ++ qdisc_pkt_len_init(skb); ++ qdisc_calculate_pkt_len(skb, q); ++ /* ++ * Heuristic to force contended enqueues to serialize on a ++ * separate lock before trying to get qdisc main lock. ++ * This permits __QDISC___STATE_RUNNING owner to get the lock more ++ * often and dequeue packets faster. ++ */ ++ contended = qdisc_is_running(q); ++ if (unlikely(contended)) ++ spin_lock(&q->busylock); ++ ++ spin_lock(root_lock); ++ if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { ++ kfree_skb(skb); ++ rc = NET_XMIT_DROP; ++ } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && ++ qdisc_run_begin(q)) { ++ /* ++ * This is a work-conserving queue; there are no old skbs ++ * waiting to be sent out; and the qdisc is not running - ++ * xmit the skb directly. ++ */ ++ ++ qdisc_bstats_update(q, skb); ++ ++ if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { ++ if (unlikely(contended)) { ++ spin_unlock(&q->busylock); ++ contended = false; ++ } ++ __qdisc_run(q); ++ } else ++ qdisc_run_end(q); ++ ++ rc = NET_XMIT_SUCCESS; ++ } else { ++ rc = q->enqueue(skb, q) & NET_XMIT_MASK; ++ if (qdisc_run_begin(q)) { ++ if (unlikely(contended)) { ++ spin_unlock(&q->busylock); ++ contended = false; ++ } ++ __qdisc_run(q); ++ } ++ } ++ spin_unlock(root_lock); ++ if (unlikely(contended)) ++ spin_unlock(&q->busylock); ++ return rc; ++} ++ ++#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) ++static void skb_update_prio(struct sk_buff *skb) ++{ ++ struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap); ++ ++ if (!skb->priority && skb->sk && map) { ++ unsigned int prioidx = skb->sk->sk_cgrp_prioidx; ++ ++ if (prioidx < map->priomap_len) ++ skb->priority = map->priomap[prioidx]; ++ } ++} ++#else ++#define skb_update_prio(skb) ++#endif ++ ++DEFINE_PER_CPU(int, xmit_recursion); ++EXPORT_SYMBOL(xmit_recursion); ++ ++#define RECURSION_LIMIT 10 ++ ++/** ++ * dev_loopback_xmit - loop back @skb ++ * @skb: buffer to transmit ++ */ ++int dev_loopback_xmit(struct sock *sk, struct sk_buff *skb) ++{ ++ skb_reset_mac_header(skb); ++ __skb_pull(skb, skb_network_offset(skb)); ++ skb->pkt_type = PACKET_LOOPBACK; ++ skb->ip_summed = CHECKSUM_UNNECESSARY; ++ WARN_ON(!skb_dst(skb)); ++ skb_dst_force(skb); ++ netif_rx_ni(skb); ++ return 0; ++} ++EXPORT_SYMBOL(dev_loopback_xmit); ++ ++/** ++ * __dev_queue_xmit - transmit a buffer ++ * @skb: buffer to transmit ++ * @accel_priv: private data used for L2 forwarding offload ++ * ++ * Queue a buffer for transmission to a network device. The caller must ++ * have set the device and priority and built the buffer before calling ++ * this function. The function can be called from an interrupt. ++ * ++ * A negative errno code is returned on a failure. A success does not ++ * guarantee the frame will be transmitted as it may be dropped due ++ * to congestion or traffic shaping. ++ * ++ * ----------------------------------------------------------------------------------- ++ * I notice this method can also return errors from the queue disciplines, ++ * including NET_XMIT_DROP, which is a positive value. So, errors can also ++ * be positive. ++ * ++ * Regardless of the return value, the skb is consumed, so it is currently ++ * difficult to retry a send to this method. (You can bump the ref count ++ * before sending to hold a reference for retry if you are careful.) ++ * ++ * When calling this method, interrupts MUST be enabled. This is because ++ * the BH enable code must have IRQs enabled so that it will not deadlock. ++ * --BLG ++ */ ++static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) ++{ ++ struct net_device *dev = skb->dev; ++ struct netdev_queue *txq; ++ struct Qdisc *q; ++ int rc = -ENOMEM; ++ ++ skb_reset_mac_header(skb); ++ ++ if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) ++ __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); ++ ++ /* Disable soft irqs for various locks below. Also ++ * stops preemption for RCU. ++ */ ++ rcu_read_lock_bh(); ++ ++ skb_update_prio(skb); ++ ++ /* If device/qdisc don't need skb->dst, release it right now while ++ * its hot in this cpu cache. ++ */ ++ if (dev->priv_flags & IFF_XMIT_DST_RELEASE) ++ skb_dst_drop(skb); ++ else ++ skb_dst_force(skb); ++ ++ txq = netdev_pick_tx(dev, skb, accel_priv); ++ q = rcu_dereference_bh(txq->qdisc); ++ ++#ifdef CONFIG_NET_CLS_ACT ++ skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS); ++#endif ++ trace_net_dev_queue(skb); ++ if (q->enqueue) { ++ rc = __dev_xmit_skb(skb, q, dev, txq); ++ goto out; ++ } ++ ++ /* The device has no queue. Common case for software devices: ++ loopback, all the sorts of tunnels... ++ ++ Really, it is unlikely that netif_tx_lock protection is necessary ++ here. (f.e. loopback and IP tunnels are clean ignoring statistics ++ counters.) ++ However, it is possible, that they rely on protection ++ made by us here. ++ ++ Check this and shot the lock. It is not prone from deadlocks. ++ Either shot noqueue qdisc, it is even simpler 8) ++ */ ++ if (dev->flags & IFF_UP) { ++ int cpu = smp_processor_id(); /* ok because BHs are off */ ++ ++ if (txq->xmit_lock_owner != cpu) { ++ ++ if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT) ++ goto recursion_alert; ++ ++ skb = validate_xmit_skb(skb, dev); ++ if (!skb) ++ goto drop; ++ ++ HARD_TX_LOCK(dev, txq, cpu); ++ ++ if (!netif_xmit_stopped(txq)) { ++ __this_cpu_inc(xmit_recursion); ++ skb = dev_hard_start_xmit(skb, dev, txq, &rc); ++ __this_cpu_dec(xmit_recursion); ++ if (dev_xmit_complete(rc)) { ++ HARD_TX_UNLOCK(dev, txq); ++ goto out; ++ } ++ } ++ HARD_TX_UNLOCK(dev, txq); ++ net_crit_ratelimited("Virtual device %s asks to queue packet!\n", ++ dev->name); ++ } else { ++ /* Recursion is detected! It is possible, ++ * unfortunately ++ */ ++recursion_alert: ++ net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", ++ dev->name); ++ } ++ } ++ ++ rc = -ENETDOWN; ++drop: ++ rcu_read_unlock_bh(); ++ ++ atomic_long_inc(&dev->tx_dropped); ++ kfree_skb_list(skb); ++ return rc; ++out: ++ rcu_read_unlock_bh(); ++ return rc; ++} ++ ++int dev_queue_xmit_sk(struct sock *sk, struct sk_buff *skb) ++{ ++ return __dev_queue_xmit(skb, NULL); ++} ++EXPORT_SYMBOL(dev_queue_xmit_sk); ++ ++int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv) ++{ ++ return __dev_queue_xmit(skb, accel_priv); ++} ++EXPORT_SYMBOL(dev_queue_xmit_accel); ++ ++ ++/*======================================================================= ++ Receiver routines ++ =======================================================================*/ ++ ++int netdev_max_backlog __read_mostly = 1000; ++EXPORT_SYMBOL(netdev_max_backlog); ++ ++int netdev_tstamp_prequeue __read_mostly = 1; ++int netdev_budget __read_mostly = 300; ++int weight_p __read_mostly = 64; /* old backlog weight */ ++ ++/* Called with irq disabled */ ++static inline void ____napi_schedule(struct softnet_data *sd, ++ struct napi_struct *napi) ++{ ++ list_add_tail(&napi->poll_list, &sd->poll_list); ++ __raise_softirq_irqoff(NET_RX_SOFTIRQ); ++} ++ ++#ifdef CONFIG_RPS ++ ++/* One global table that all flow-based protocols share. */ ++struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; ++EXPORT_SYMBOL(rps_sock_flow_table); ++u32 rps_cpu_mask __read_mostly; ++EXPORT_SYMBOL(rps_cpu_mask); ++ ++struct static_key rps_needed __read_mostly; ++ ++static struct rps_dev_flow * ++set_rps_cpu(struct net_device *dev, struct sk_buff *skb, ++ struct rps_dev_flow *rflow, u16 next_cpu) ++{ ++ if (next_cpu < nr_cpu_ids) { ++#ifdef CONFIG_RFS_ACCEL ++ struct netdev_rx_queue *rxqueue; ++ struct rps_dev_flow_table *flow_table; ++ struct rps_dev_flow *old_rflow; ++ u32 flow_id; ++ u16 rxq_index; ++ int rc; ++ ++ /* Should we steer this flow to a different hardware queue? */ ++ if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || ++ !(dev->features & NETIF_F_NTUPLE)) ++ goto out; ++ rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); ++ if (rxq_index == skb_get_rx_queue(skb)) ++ goto out; ++ ++ rxqueue = dev->_rx + rxq_index; ++ flow_table = rcu_dereference(rxqueue->rps_flow_table); ++ if (!flow_table) ++ goto out; ++ flow_id = skb_get_hash(skb) & flow_table->mask; ++ rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, ++ rxq_index, flow_id); ++ if (rc < 0) ++ goto out; ++ old_rflow = rflow; ++ rflow = &flow_table->flows[flow_id]; ++ rflow->filter = rc; ++ if (old_rflow->filter == rflow->filter) ++ old_rflow->filter = RPS_NO_FILTER; ++ out: ++#endif ++ rflow->last_qtail = ++ per_cpu(softnet_data, next_cpu).input_queue_head; ++ } ++ ++ rflow->cpu = next_cpu; ++ return rflow; ++} ++ ++/* ++ * get_rps_cpu is called from netif_receive_skb and returns the target ++ * CPU from the RPS map of the receiving queue for a given skb. ++ * rcu_read_lock must be held on entry. ++ */ ++static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, ++ struct rps_dev_flow **rflowp) ++{ ++ const struct rps_sock_flow_table *sock_flow_table; ++ struct netdev_rx_queue *rxqueue = dev->_rx; ++ struct rps_dev_flow_table *flow_table; ++ struct rps_map *map; ++ int cpu = -1; ++ u32 tcpu; ++ u32 hash; ++ ++ if (skb_rx_queue_recorded(skb)) { ++ u16 index = skb_get_rx_queue(skb); ++ ++ if (unlikely(index >= dev->real_num_rx_queues)) { ++ WARN_ONCE(dev->real_num_rx_queues > 1, ++ "%s received packet on queue %u, but number " ++ "of RX queues is %u\n", ++ dev->name, index, dev->real_num_rx_queues); ++ goto done; ++ } ++ rxqueue += index; ++ } ++ ++ /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ ++ ++ flow_table = rcu_dereference(rxqueue->rps_flow_table); ++ map = rcu_dereference(rxqueue->rps_map); ++ if (!flow_table && !map) ++ goto done; ++ ++ skb_reset_network_header(skb); ++ hash = skb_get_hash(skb); ++ if (!hash) ++ goto done; ++ ++ sock_flow_table = rcu_dereference(rps_sock_flow_table); ++ if (flow_table && sock_flow_table) { ++ struct rps_dev_flow *rflow; ++ u32 next_cpu; ++ u32 ident; ++ ++ /* First check into global flow table if there is a match */ ++ ident = sock_flow_table->ents[hash & sock_flow_table->mask]; ++ if ((ident ^ hash) & ~rps_cpu_mask) ++ goto try_rps; ++ ++ next_cpu = ident & rps_cpu_mask; ++ ++ /* OK, now we know there is a match, ++ * we can look at the local (per receive queue) flow table ++ */ ++ rflow = &flow_table->flows[hash & flow_table->mask]; ++ tcpu = rflow->cpu; ++ ++ /* ++ * If the desired CPU (where last recvmsg was done) is ++ * different from current CPU (one in the rx-queue flow ++ * table entry), switch if one of the following holds: ++ * - Current CPU is unset (>= nr_cpu_ids). ++ * - Current CPU is offline. ++ * - The current CPU's queue tail has advanced beyond the ++ * last packet that was enqueued using this table entry. ++ * This guarantees that all previous packets for the flow ++ * have been dequeued, thus preserving in order delivery. ++ */ ++ if (unlikely(tcpu != next_cpu) && ++ (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || ++ ((int)(per_cpu(softnet_data, tcpu).input_queue_head - ++ rflow->last_qtail)) >= 0)) { ++ tcpu = next_cpu; ++ rflow = set_rps_cpu(dev, skb, rflow, next_cpu); ++ } ++ ++ if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { ++ *rflowp = rflow; ++ cpu = tcpu; ++ goto done; ++ } ++ } ++ ++try_rps: ++ ++ if (map) { ++ tcpu = map->cpus[reciprocal_scale(hash, map->len)]; ++ if (cpu_online(tcpu)) { ++ cpu = tcpu; ++ goto done; ++ } ++ } ++ ++done: ++ return cpu; ++} ++ ++#ifdef CONFIG_RFS_ACCEL ++ ++/** ++ * rps_may_expire_flow - check whether an RFS hardware filter may be removed ++ * @dev: Device on which the filter was set ++ * @rxq_index: RX queue index ++ * @flow_id: Flow ID passed to ndo_rx_flow_steer() ++ * @filter_id: Filter ID returned by ndo_rx_flow_steer() ++ * ++ * Drivers that implement ndo_rx_flow_steer() should periodically call ++ * this function for each installed filter and remove the filters for ++ * which it returns %true. ++ */ ++bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, ++ u32 flow_id, u16 filter_id) ++{ ++ struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; ++ struct rps_dev_flow_table *flow_table; ++ struct rps_dev_flow *rflow; ++ bool expire = true; ++ unsigned int cpu; ++ ++ rcu_read_lock(); ++ flow_table = rcu_dereference(rxqueue->rps_flow_table); ++ if (flow_table && flow_id <= flow_table->mask) { ++ rflow = &flow_table->flows[flow_id]; ++ cpu = ACCESS_ONCE(rflow->cpu); ++ if (rflow->filter == filter_id && cpu < nr_cpu_ids && ++ ((int)(per_cpu(softnet_data, cpu).input_queue_head - ++ rflow->last_qtail) < ++ (int)(10 * flow_table->mask))) ++ expire = false; ++ } ++ rcu_read_unlock(); ++ return expire; ++} ++EXPORT_SYMBOL(rps_may_expire_flow); ++ ++#endif /* CONFIG_RFS_ACCEL */ ++ ++/* Called from hardirq (IPI) context */ ++static void rps_trigger_softirq(void *data) ++{ ++ struct softnet_data *sd = data; ++ ++ ____napi_schedule(sd, &sd->backlog); ++ sd->received_rps++; ++} ++ ++#endif /* CONFIG_RPS */ ++ ++/* ++ * Check if this softnet_data structure is another cpu one ++ * If yes, queue it to our IPI list and return 1 ++ * If no, return 0 ++ */ ++static int rps_ipi_queued(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ struct softnet_data *mysd = this_cpu_ptr(&softnet_data); ++ ++ if (sd != mysd) { ++ sd->rps_ipi_next = mysd->rps_ipi_list; ++ mysd->rps_ipi_list = sd; ++ ++ __raise_softirq_irqoff(NET_RX_SOFTIRQ); ++ return 1; ++ } ++#endif /* CONFIG_RPS */ ++ return 0; ++} ++ ++#ifdef CONFIG_NET_FLOW_LIMIT ++int netdev_flow_limit_table_len __read_mostly = (1 << 12); ++#endif ++ ++static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) ++{ ++#ifdef CONFIG_NET_FLOW_LIMIT ++ struct sd_flow_limit *fl; ++ struct softnet_data *sd; ++ unsigned int old_flow, new_flow; ++ ++ if (qlen < (netdev_max_backlog >> 1)) ++ return false; ++ ++ sd = this_cpu_ptr(&softnet_data); ++ ++ rcu_read_lock(); ++ fl = rcu_dereference(sd->flow_limit); ++ if (fl) { ++ new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); ++ old_flow = fl->history[fl->history_head]; ++ fl->history[fl->history_head] = new_flow; ++ ++ fl->history_head++; ++ fl->history_head &= FLOW_LIMIT_HISTORY - 1; ++ ++ if (likely(fl->buckets[old_flow])) ++ fl->buckets[old_flow]--; ++ ++ if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { ++ fl->count++; ++ rcu_read_unlock(); ++ return true; ++ } ++ } ++ rcu_read_unlock(); ++#endif ++ return false; ++} ++ ++/* ++ * enqueue_to_backlog is called to queue an skb to a per CPU backlog ++ * queue (may be a remote CPU queue). ++ */ ++static int enqueue_to_backlog(struct sk_buff *skb, int cpu, ++ unsigned int *qtail) ++{ ++ struct softnet_data *sd; ++ unsigned long flags; ++ unsigned int qlen; ++ ++ sd = &per_cpu(softnet_data, cpu); ++ ++ local_irq_save(flags); ++ ++ rps_lock(sd); ++ if (!netif_running(skb->dev)) ++ goto drop; ++ qlen = skb_queue_len(&sd->input_pkt_queue); ++ if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { ++ if (qlen) { ++enqueue: ++ __skb_queue_tail(&sd->input_pkt_queue, skb); ++ input_queue_tail_incr_save(sd, qtail); ++ rps_unlock(sd); ++ local_irq_restore(flags); ++ return NET_RX_SUCCESS; ++ } ++ ++ /* Schedule NAPI for backlog device ++ * We can use non atomic operation since we own the queue lock ++ */ ++ if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { ++ if (!rps_ipi_queued(sd)) ++ ____napi_schedule(sd, &sd->backlog); ++ } ++ goto enqueue; ++ } ++ ++drop: ++ sd->dropped++; ++ rps_unlock(sd); ++ ++ local_irq_restore(flags); ++ ++ atomic_long_inc(&skb->dev->rx_dropped); ++ kfree_skb(skb); ++ return NET_RX_DROP; ++} ++ ++static int netif_rx_internal(struct sk_buff *skb) ++{ ++ int ret; ++ ++ net_timestamp_check(netdev_tstamp_prequeue, skb); ++ ++ trace_netif_rx(skb); ++#ifdef CONFIG_RPS ++ if (static_key_false(&rps_needed)) { ++ struct rps_dev_flow voidflow, *rflow = &voidflow; ++ int cpu; ++ ++ preempt_disable(); ++ rcu_read_lock(); ++ ++ cpu = get_rps_cpu(skb->dev, skb, &rflow); ++ if (cpu < 0) ++ cpu = smp_processor_id(); ++ ++ ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); ++ ++ rcu_read_unlock(); ++ preempt_enable(); ++ } else ++#endif ++ { ++ unsigned int qtail; ++ ret = enqueue_to_backlog(skb, get_cpu(), &qtail); ++ put_cpu(); ++ } ++ return ret; ++} ++ ++/** ++ * netif_rx - post buffer to the network code ++ * @skb: buffer to post ++ * ++ * This function receives a packet from a device driver and queues it for ++ * the upper (protocol) levels to process. It always succeeds. The buffer ++ * may be dropped during processing for congestion control or by the ++ * protocol layers. ++ * ++ * return values: ++ * NET_RX_SUCCESS (no congestion) ++ * NET_RX_DROP (packet was dropped) ++ * ++ */ ++ ++int netif_rx(struct sk_buff *skb) ++{ ++ trace_netif_rx_entry(skb); ++ ++ return netif_rx_internal(skb); ++} ++EXPORT_SYMBOL(netif_rx); ++ ++int netif_rx_ni(struct sk_buff *skb) ++{ ++ int err; ++ ++ trace_netif_rx_ni_entry(skb); ++ ++ preempt_disable(); ++ err = netif_rx_internal(skb); ++ if (local_softirq_pending()) ++ do_softirq(); ++ preempt_enable(); ++ ++ return err; ++} ++EXPORT_SYMBOL(netif_rx_ni); ++ ++static void net_tx_action(struct softirq_action *h) ++{ ++ struct softnet_data *sd = this_cpu_ptr(&softnet_data); ++ ++ if (sd->completion_queue) { ++ struct sk_buff *clist; ++ ++ local_irq_disable(); ++ clist = sd->completion_queue; ++ sd->completion_queue = NULL; ++ local_irq_enable(); ++ ++ while (clist) { ++ struct sk_buff *skb = clist; ++ clist = clist->next; ++ ++ WARN_ON(atomic_read(&skb->users)); ++ if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) ++ trace_consume_skb(skb); ++ else ++ trace_kfree_skb(skb, net_tx_action); ++ __kfree_skb(skb); ++ } ++ } ++ ++ if (sd->output_queue) { ++ struct Qdisc *head; ++ ++ local_irq_disable(); ++ head = sd->output_queue; ++ sd->output_queue = NULL; ++ sd->output_queue_tailp = &sd->output_queue; ++ local_irq_enable(); ++ ++ while (head) { ++ struct Qdisc *q = head; ++ spinlock_t *root_lock; ++ ++ head = head->next_sched; ++ ++ root_lock = qdisc_lock(q); ++ if (spin_trylock(root_lock)) { ++ smp_mb__before_atomic(); ++ clear_bit(__QDISC_STATE_SCHED, ++ &q->state); ++ qdisc_run(q); ++ spin_unlock(root_lock); ++ } else { ++ if (!test_bit(__QDISC_STATE_DEACTIVATED, ++ &q->state)) { ++ __netif_reschedule(q); ++ } else { ++ smp_mb__before_atomic(); ++ clear_bit(__QDISC_STATE_SCHED, ++ &q->state); ++ } ++ } ++ } ++ } ++} ++ ++#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \ ++ (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE)) ++/* This hook is defined here for ATM LANE */ ++int (*br_fdb_test_addr_hook)(struct net_device *dev, ++ unsigned char *addr) __read_mostly; ++EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); ++#endif ++ ++#ifdef CONFIG_NET_CLS_ACT ++/* TODO: Maybe we should just force sch_ingress to be compiled in ++ * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions ++ * a compare and 2 stores extra right now if we dont have it on ++ * but have CONFIG_NET_CLS_ACT ++ * NOTE: This doesn't stop any functionality; if you dont have ++ * the ingress scheduler, you just can't add policies on ingress. ++ * ++ */ ++static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq) ++{ ++ struct net_device *dev = skb->dev; ++ u32 ttl = G_TC_RTTL(skb->tc_verd); ++ int result = TC_ACT_OK; ++ struct Qdisc *q; ++ ++ if (unlikely(MAX_RED_LOOP < ttl++)) { ++ net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n", ++ skb->skb_iif, dev->ifindex); ++ return TC_ACT_SHOT; ++ } ++ ++ skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl); ++ skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS); ++ ++ q = rcu_dereference(rxq->qdisc); ++ if (q != &noop_qdisc) { ++ spin_lock(qdisc_lock(q)); ++ if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) ++ result = qdisc_enqueue_root(skb, q); ++ spin_unlock(qdisc_lock(q)); ++ } ++ ++ return result; ++} ++ ++static inline struct sk_buff *handle_ing(struct sk_buff *skb, ++ struct packet_type **pt_prev, ++ int *ret, struct net_device *orig_dev) ++{ ++ struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue); ++ ++ if (!rxq || rcu_access_pointer(rxq->qdisc) == &noop_qdisc) ++ return skb; ++ ++ if (*pt_prev) { ++ *ret = deliver_skb(skb, *pt_prev, orig_dev); ++ *pt_prev = NULL; ++ } ++ ++ switch (ing_filter(skb, rxq)) { ++ case TC_ACT_SHOT: ++ case TC_ACT_STOLEN: ++ kfree_skb(skb); ++ return NULL; ++ } ++ ++ return skb; ++} ++#endif ++ ++/** ++ * netdev_rx_handler_register - register receive handler ++ * @dev: device to register a handler for ++ * @rx_handler: receive handler to register ++ * @rx_handler_data: data pointer that is used by rx handler ++ * ++ * Register a receive handler for a device. This handler will then be ++ * called from __netif_receive_skb. A negative errno code is returned ++ * on a failure. ++ * ++ * The caller must hold the rtnl_mutex. ++ * ++ * For a general description of rx_handler, see enum rx_handler_result. ++ */ ++int netdev_rx_handler_register(struct net_device *dev, ++ rx_handler_func_t *rx_handler, ++ void *rx_handler_data) ++{ ++ ASSERT_RTNL(); ++ ++ if (dev->rx_handler) ++ return -EBUSY; ++ ++ /* Note: rx_handler_data must be set before rx_handler */ ++ rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); ++ rcu_assign_pointer(dev->rx_handler, rx_handler); ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(netdev_rx_handler_register); ++ ++/** ++ * netdev_rx_handler_unregister - unregister receive handler ++ * @dev: device to unregister a handler from ++ * ++ * Unregister a receive handler from a device. ++ * ++ * The caller must hold the rtnl_mutex. ++ */ ++void netdev_rx_handler_unregister(struct net_device *dev) ++{ ++ ++ ASSERT_RTNL(); ++ RCU_INIT_POINTER(dev->rx_handler, NULL); ++ /* a reader seeing a non NULL rx_handler in a rcu_read_lock() ++ * section has a guarantee to see a non NULL rx_handler_data ++ * as well. ++ */ ++ synchronize_net(); ++ RCU_INIT_POINTER(dev->rx_handler_data, NULL); ++} ++EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); ++ ++/* ++ * Limit the use of PFMEMALLOC reserves to those protocols that implement ++ * the special handling of PFMEMALLOC skbs. ++ */ ++static bool skb_pfmemalloc_protocol(struct sk_buff *skb) ++{ ++ switch (skb->protocol) { ++ case htons(ETH_P_ARP): ++ case htons(ETH_P_IP): ++ case htons(ETH_P_IPV6): ++ case htons(ETH_P_8021Q): ++ case htons(ETH_P_8021AD): ++ return true; ++ default: ++ return false; ++ } ++} ++ ++static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) ++{ ++ struct packet_type *ptype, *pt_prev; ++ rx_handler_func_t *rx_handler; ++ struct net_device *orig_dev; ++ bool deliver_exact = false; ++ int ret = NET_RX_DROP; ++ __be16 type; ++ ++ net_timestamp_check(!netdev_tstamp_prequeue, skb); ++ ++ trace_netif_receive_skb(skb); ++ ++ orig_dev = skb->dev; ++ ++ skb_reset_network_header(skb); ++ if (!skb_transport_header_was_set(skb)) ++ skb_reset_transport_header(skb); ++ skb_reset_mac_len(skb); ++ ++ pt_prev = NULL; ++ ++another_round: ++ skb->skb_iif = skb->dev->ifindex; ++ ++ __this_cpu_inc(softnet_data.processed); ++ ++ if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || ++ skb->protocol == cpu_to_be16(ETH_P_8021AD)) { ++ skb = skb_vlan_untag(skb); ++ if (unlikely(!skb)) ++ goto out; ++ } ++ ++#ifdef CONFIG_NET_CLS_ACT ++ if (skb->tc_verd & TC_NCLS) { ++ skb->tc_verd = CLR_TC_NCLS(skb->tc_verd); ++ goto ncls; ++ } ++#endif ++ ++ if (pfmemalloc) ++ goto skip_taps; ++ ++ list_for_each_entry_rcu(ptype, &ptype_all, list) { ++ if (pt_prev) ++ ret = deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = ptype; ++ } ++ ++ list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { ++ if (pt_prev) ++ ret = deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = ptype; ++ } ++ ++skip_taps: ++#ifdef CONFIG_NET_CLS_ACT ++ if (static_key_false(&ingress_needed)) { ++ skb = handle_ing(skb, &pt_prev, &ret, orig_dev); ++ if (!skb) ++ goto out; ++ } ++ ++ skb->tc_verd = 0; ++ncls: ++#endif ++ if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) ++ goto drop; ++ ++ if (skb_vlan_tag_present(skb)) { ++ if (pt_prev) { ++ ret = deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = NULL; ++ } ++ if (vlan_do_receive(&skb)) ++ goto another_round; ++ else if (unlikely(!skb)) ++ goto out; ++ } ++ ++ rx_handler = rcu_dereference(skb->dev->rx_handler); ++ if (rx_handler) { ++ if (pt_prev) { ++ ret = deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = NULL; ++ } ++ switch (rx_handler(&skb)) { ++ case RX_HANDLER_CONSUMED: ++ ret = NET_RX_SUCCESS; ++ goto out; ++ case RX_HANDLER_ANOTHER: ++ goto another_round; ++ case RX_HANDLER_EXACT: ++ deliver_exact = true; ++ case RX_HANDLER_PASS: ++ break; ++ default: ++ BUG(); ++ } ++ } ++ ++ if (unlikely(skb_vlan_tag_present(skb))) { ++ if (skb_vlan_tag_get_id(skb)) ++ skb->pkt_type = PACKET_OTHERHOST; ++ /* Note: we might in the future use prio bits ++ * and set skb->priority like in vlan_do_receive() ++ * For the time being, just ignore Priority Code Point ++ */ ++ skb->vlan_tci = 0; ++ } ++ ++ type = skb->protocol; ++ ++ /* deliver only exact match when indicated */ ++ if (likely(!deliver_exact)) { ++ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, ++ &ptype_base[ntohs(type) & ++ PTYPE_HASH_MASK]); ++ } ++ ++ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, ++ &orig_dev->ptype_specific); ++ ++ if (unlikely(skb->dev != orig_dev)) { ++ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, ++ &skb->dev->ptype_specific); ++ } ++ ++ if (pt_prev) { ++ if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) ++ goto drop; ++ else ++ ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); ++ } else { ++drop: ++ atomic_long_inc(&skb->dev->rx_dropped); ++ kfree_skb(skb); ++ /* Jamal, now you will not able to escape explaining ++ * me how you were going to use this. :-) ++ */ ++ ret = NET_RX_DROP; ++ } ++ ++out: ++ return ret; ++} ++ ++static int __netif_receive_skb(struct sk_buff *skb) ++{ ++ int ret; ++ ++ if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { ++ unsigned long pflags = current->flags; ++ ++ /* ++ * PFMEMALLOC skbs are special, they should ++ * - be delivered to SOCK_MEMALLOC sockets only ++ * - stay away from userspace ++ * - have bounded memory usage ++ * ++ * Use PF_MEMALLOC as this saves us from propagating the allocation ++ * context down to all allocation sites. ++ */ ++ current->flags |= PF_MEMALLOC; ++ ret = __netif_receive_skb_core(skb, true); ++ tsk_restore_flags(current, pflags, PF_MEMALLOC); ++ } else ++ ret = __netif_receive_skb_core(skb, false); ++ ++ return ret; ++} ++ ++static int netif_receive_skb_internal(struct sk_buff *skb) ++{ ++ int ret; ++ ++ net_timestamp_check(netdev_tstamp_prequeue, skb); ++ ++ if (skb_defer_rx_timestamp(skb)) ++ return NET_RX_SUCCESS; ++ ++ rcu_read_lock(); ++ ++#ifdef CONFIG_RPS ++ if (static_key_false(&rps_needed)) { ++ struct rps_dev_flow voidflow, *rflow = &voidflow; ++ int cpu = get_rps_cpu(skb->dev, skb, &rflow); ++ ++ if (cpu >= 0) { ++ ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); ++ rcu_read_unlock(); ++ return ret; ++ } ++ } ++#endif ++ ret = __netif_receive_skb(skb); ++ rcu_read_unlock(); ++ return ret; ++} ++ ++/** ++ * netif_receive_skb - process receive buffer from network ++ * @skb: buffer to process ++ * ++ * netif_receive_skb() is the main receive data processing function. ++ * It always succeeds. The buffer may be dropped during processing ++ * for congestion control or by the protocol layers. ++ * ++ * This function may only be called from softirq context and interrupts ++ * should be enabled. ++ * ++ * Return values (usually ignored): ++ * NET_RX_SUCCESS: no congestion ++ * NET_RX_DROP: packet was dropped ++ */ ++int netif_receive_skb_sk(struct sock *sk, struct sk_buff *skb) ++{ ++ trace_netif_receive_skb_entry(skb); ++ ++ return netif_receive_skb_internal(skb); ++} ++EXPORT_SYMBOL(netif_receive_skb_sk); ++ ++/* Network device is going away, flush any packets still pending ++ * Called with irqs disabled. ++ */ ++static void flush_backlog(void *arg) ++{ ++ struct net_device *dev = arg; ++ struct softnet_data *sd = this_cpu_ptr(&softnet_data); ++ struct sk_buff *skb, *tmp; ++ ++ rps_lock(sd); ++ skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { ++ if (skb->dev == dev) { ++ __skb_unlink(skb, &sd->input_pkt_queue); ++ kfree_skb(skb); ++ input_queue_head_incr(sd); ++ } ++ } ++ rps_unlock(sd); ++ ++ skb_queue_walk_safe(&sd->process_queue, skb, tmp) { ++ if (skb->dev == dev) { ++ __skb_unlink(skb, &sd->process_queue); ++ kfree_skb(skb); ++ input_queue_head_incr(sd); ++ } ++ } ++} ++ ++static int napi_gro_complete(struct sk_buff *skb) ++{ ++ struct packet_offload *ptype; ++ __be16 type = skb->protocol; ++ struct list_head *head = &offload_base; ++ int err = -ENOENT; ++ ++ BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); ++ ++ if (NAPI_GRO_CB(skb)->count == 1) { ++ skb_shinfo(skb)->gso_size = 0; ++ goto out; ++ } ++ ++ rcu_read_lock(); ++ list_for_each_entry_rcu(ptype, head, list) { ++ if (ptype->type != type || !ptype->callbacks.gro_complete) ++ continue; ++ ++ err = ptype->callbacks.gro_complete(skb, 0); ++ break; ++ } ++ rcu_read_unlock(); ++ ++ if (err) { ++ WARN_ON(&ptype->list == head); ++ kfree_skb(skb); ++ return NET_RX_SUCCESS; ++ } ++ ++out: ++ return netif_receive_skb_internal(skb); ++} ++ ++/* napi->gro_list contains packets ordered by age. ++ * youngest packets at the head of it. ++ * Complete skbs in reverse order to reduce latencies. ++ */ ++void napi_gro_flush(struct napi_struct *napi, bool flush_old) ++{ ++ struct sk_buff *skb, *prev = NULL; ++ ++ /* scan list and build reverse chain */ ++ for (skb = napi->gro_list; skb != NULL; skb = skb->next) { ++ skb->prev = prev; ++ prev = skb; ++ } ++ ++ for (skb = prev; skb; skb = prev) { ++ skb->next = NULL; ++ ++ if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) ++ return; ++ ++ prev = skb->prev; ++ napi_gro_complete(skb); ++ napi->gro_count--; ++ } ++ ++ napi->gro_list = NULL; ++} ++EXPORT_SYMBOL(napi_gro_flush); ++ ++static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb) ++{ ++ struct sk_buff *p; ++ unsigned int maclen = skb->dev->hard_header_len; ++ u32 hash = skb_get_hash_raw(skb); ++ ++ for (p = napi->gro_list; p; p = p->next) { ++ unsigned long diffs; ++ ++ NAPI_GRO_CB(p)->flush = 0; ++ ++ if (hash != skb_get_hash_raw(p)) { ++ NAPI_GRO_CB(p)->same_flow = 0; ++ continue; ++ } ++ ++ diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; ++ diffs |= p->vlan_tci ^ skb->vlan_tci; ++ if (maclen == ETH_HLEN) ++ diffs |= compare_ether_header(skb_mac_header(p), ++ skb_mac_header(skb)); ++ else if (!diffs) ++ diffs = memcmp(skb_mac_header(p), ++ skb_mac_header(skb), ++ maclen); ++ NAPI_GRO_CB(p)->same_flow = !diffs; ++ } ++} ++ ++static void skb_gro_reset_offset(struct sk_buff *skb) ++{ ++ const struct skb_shared_info *pinfo = skb_shinfo(skb); ++ const skb_frag_t *frag0 = &pinfo->frags[0]; ++ ++ NAPI_GRO_CB(skb)->data_offset = 0; ++ NAPI_GRO_CB(skb)->frag0 = NULL; ++ NAPI_GRO_CB(skb)->frag0_len = 0; ++ ++ if (skb_mac_header(skb) == skb_tail_pointer(skb) && ++ pinfo->nr_frags && ++ !PageHighMem(skb_frag_page(frag0))) { ++ NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); ++ NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0); ++ } ++} ++ ++static void gro_pull_from_frag0(struct sk_buff *skb, int grow) ++{ ++ struct skb_shared_info *pinfo = skb_shinfo(skb); ++ ++ BUG_ON(skb->end - skb->tail < grow); ++ ++ memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); ++ ++ skb->data_len -= grow; ++ skb->tail += grow; ++ ++ pinfo->frags[0].page_offset += grow; ++ skb_frag_size_sub(&pinfo->frags[0], grow); ++ ++ if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { ++ skb_frag_unref(skb, 0); ++ memmove(pinfo->frags, pinfo->frags + 1, ++ --pinfo->nr_frags * sizeof(pinfo->frags[0])); ++ } ++} ++ ++static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) ++{ ++ struct sk_buff **pp = NULL; ++ struct packet_offload *ptype; ++ __be16 type = skb->protocol; ++ struct list_head *head = &offload_base; ++ int same_flow; ++ enum gro_result ret; ++ int grow; ++ ++ if (!(skb->dev->features & NETIF_F_GRO)) ++ goto normal; ++ ++ if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad) ++ goto normal; ++ ++ gro_list_prepare(napi, skb); ++ ++ rcu_read_lock(); ++ list_for_each_entry_rcu(ptype, head, list) { ++ if (ptype->type != type || !ptype->callbacks.gro_receive) ++ continue; ++ ++ skb_set_network_header(skb, skb_gro_offset(skb)); ++ skb_reset_mac_len(skb); ++ NAPI_GRO_CB(skb)->same_flow = 0; ++ NAPI_GRO_CB(skb)->flush = 0; ++ NAPI_GRO_CB(skb)->free = 0; ++ NAPI_GRO_CB(skb)->udp_mark = 0; ++ NAPI_GRO_CB(skb)->gro_remcsum_start = 0; ++ ++ /* Setup for GRO checksum validation */ ++ switch (skb->ip_summed) { ++ case CHECKSUM_COMPLETE: ++ NAPI_GRO_CB(skb)->csum = skb->csum; ++ NAPI_GRO_CB(skb)->csum_valid = 1; ++ NAPI_GRO_CB(skb)->csum_cnt = 0; ++ break; ++ case CHECKSUM_UNNECESSARY: ++ NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; ++ NAPI_GRO_CB(skb)->csum_valid = 0; ++ break; ++ default: ++ NAPI_GRO_CB(skb)->csum_cnt = 0; ++ NAPI_GRO_CB(skb)->csum_valid = 0; ++ } ++ ++ pp = ptype->callbacks.gro_receive(&napi->gro_list, skb); ++ break; ++ } ++ rcu_read_unlock(); ++ ++ if (&ptype->list == head) ++ goto normal; ++ ++ same_flow = NAPI_GRO_CB(skb)->same_flow; ++ ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; ++ ++ if (pp) { ++ struct sk_buff *nskb = *pp; ++ ++ *pp = nskb->next; ++ nskb->next = NULL; ++ napi_gro_complete(nskb); ++ napi->gro_count--; ++ } ++ ++ if (same_flow) ++ goto ok; ++ ++ if (NAPI_GRO_CB(skb)->flush) ++ goto normal; ++ ++ if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) { ++ struct sk_buff *nskb = napi->gro_list; ++ ++ /* locate the end of the list to select the 'oldest' flow */ ++ while (nskb->next) { ++ pp = &nskb->next; ++ nskb = *pp; ++ } ++ *pp = NULL; ++ nskb->next = NULL; ++ napi_gro_complete(nskb); ++ } else { ++ napi->gro_count++; ++ } ++ NAPI_GRO_CB(skb)->count = 1; ++ NAPI_GRO_CB(skb)->age = jiffies; ++ NAPI_GRO_CB(skb)->last = skb; ++ skb_shinfo(skb)->gso_size = skb_gro_len(skb); ++ skb->next = napi->gro_list; ++ napi->gro_list = skb; ++ ret = GRO_HELD; ++ ++pull: ++ grow = skb_gro_offset(skb) - skb_headlen(skb); ++ if (grow > 0) ++ gro_pull_from_frag0(skb, grow); ++ok: ++ return ret; ++ ++normal: ++ ret = GRO_NORMAL; ++ goto pull; ++} ++ ++struct packet_offload *gro_find_receive_by_type(__be16 type) ++{ ++ struct list_head *offload_head = &offload_base; ++ struct packet_offload *ptype; ++ ++ list_for_each_entry_rcu(ptype, offload_head, list) { ++ if (ptype->type != type || !ptype->callbacks.gro_receive) ++ continue; ++ return ptype; ++ } ++ return NULL; ++} ++EXPORT_SYMBOL(gro_find_receive_by_type); ++ ++struct packet_offload *gro_find_complete_by_type(__be16 type) ++{ ++ struct list_head *offload_head = &offload_base; ++ struct packet_offload *ptype; ++ ++ list_for_each_entry_rcu(ptype, offload_head, list) { ++ if (ptype->type != type || !ptype->callbacks.gro_complete) ++ continue; ++ return ptype; ++ } ++ return NULL; ++} ++EXPORT_SYMBOL(gro_find_complete_by_type); ++ ++static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) ++{ ++ switch (ret) { ++ case GRO_NORMAL: ++ if (netif_receive_skb_internal(skb)) ++ ret = GRO_DROP; ++ break; ++ ++ case GRO_DROP: ++ kfree_skb(skb); ++ break; ++ ++ case GRO_MERGED_FREE: ++ if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) ++ kmem_cache_free(skbuff_head_cache, skb); ++ else ++ __kfree_skb(skb); ++ break; ++ ++ case GRO_HELD: ++ case GRO_MERGED: ++ break; ++ } ++ ++ return ret; ++} ++ ++gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) ++{ ++ trace_napi_gro_receive_entry(skb); ++ ++ skb_gro_reset_offset(skb); ++ ++ return napi_skb_finish(dev_gro_receive(napi, skb), skb); ++} ++EXPORT_SYMBOL(napi_gro_receive); ++ ++static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) ++{ ++ if (unlikely(skb->pfmemalloc)) { ++ consume_skb(skb); ++ return; ++ } ++ __skb_pull(skb, skb_headlen(skb)); ++ /* restore the reserve we had after netdev_alloc_skb_ip_align() */ ++ skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); ++ skb->vlan_tci = 0; ++ skb->dev = napi->dev; ++ skb->skb_iif = 0; ++ skb->encapsulation = 0; ++ skb_shinfo(skb)->gso_type = 0; ++ skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); ++ ++ napi->skb = skb; ++} ++ ++struct sk_buff *napi_get_frags(struct napi_struct *napi) ++{ ++ struct sk_buff *skb = napi->skb; ++ ++ if (!skb) { ++ skb = napi_alloc_skb(napi, GRO_MAX_HEAD); ++ napi->skb = skb; ++ } ++ return skb; ++} ++EXPORT_SYMBOL(napi_get_frags); ++ ++static gro_result_t napi_frags_finish(struct napi_struct *napi, ++ struct sk_buff *skb, ++ gro_result_t ret) ++{ ++ switch (ret) { ++ case GRO_NORMAL: ++ case GRO_HELD: ++ __skb_push(skb, ETH_HLEN); ++ skb->protocol = eth_type_trans(skb, skb->dev); ++ if (ret == GRO_NORMAL && netif_receive_skb_internal(skb)) ++ ret = GRO_DROP; ++ break; ++ ++ case GRO_DROP: ++ case GRO_MERGED_FREE: ++ napi_reuse_skb(napi, skb); ++ break; ++ ++ case GRO_MERGED: ++ break; ++ } ++ ++ return ret; ++} ++ ++/* Upper GRO stack assumes network header starts at gro_offset=0 ++ * Drivers could call both napi_gro_frags() and napi_gro_receive() ++ * We copy ethernet header into skb->data to have a common layout. ++ */ ++static struct sk_buff *napi_frags_skb(struct napi_struct *napi) ++{ ++ struct sk_buff *skb = napi->skb; ++ const struct ethhdr *eth; ++ unsigned int hlen = sizeof(*eth); ++ ++ napi->skb = NULL; ++ ++ skb_reset_mac_header(skb); ++ skb_gro_reset_offset(skb); ++ ++ eth = skb_gro_header_fast(skb, 0); ++ if (unlikely(skb_gro_header_hard(skb, hlen))) { ++ eth = skb_gro_header_slow(skb, hlen, 0); ++ if (unlikely(!eth)) { ++ napi_reuse_skb(napi, skb); ++ return NULL; ++ } ++ } else { ++ gro_pull_from_frag0(skb, hlen); ++ NAPI_GRO_CB(skb)->frag0 += hlen; ++ NAPI_GRO_CB(skb)->frag0_len -= hlen; ++ } ++ __skb_pull(skb, hlen); ++ ++ /* ++ * This works because the only protocols we care about don't require ++ * special handling. ++ * We'll fix it up properly in napi_frags_finish() ++ */ ++ skb->protocol = eth->h_proto; ++ ++ return skb; ++} ++ ++gro_result_t napi_gro_frags(struct napi_struct *napi) ++{ ++ struct sk_buff *skb = napi_frags_skb(napi); ++ ++ if (!skb) ++ return GRO_DROP; ++ ++ trace_napi_gro_frags_entry(skb); ++ ++ return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); ++} ++EXPORT_SYMBOL(napi_gro_frags); ++ ++/* Compute the checksum from gro_offset and return the folded value ++ * after adding in any pseudo checksum. ++ */ ++__sum16 __skb_gro_checksum_complete(struct sk_buff *skb) ++{ ++ __wsum wsum; ++ __sum16 sum; ++ ++ wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); ++ ++ /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ ++ sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); ++ if (likely(!sum)) { ++ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && ++ !skb->csum_complete_sw) ++ netdev_rx_csum_fault(skb->dev); ++ } ++ ++ NAPI_GRO_CB(skb)->csum = wsum; ++ NAPI_GRO_CB(skb)->csum_valid = 1; ++ ++ return sum; ++} ++EXPORT_SYMBOL(__skb_gro_checksum_complete); ++ ++/* ++ * net_rps_action_and_irq_enable sends any pending IPI's for rps. ++ * Note: called with local irq disabled, but exits with local irq enabled. ++ */ ++static void net_rps_action_and_irq_enable(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ struct softnet_data *remsd = sd->rps_ipi_list; ++ ++ if (remsd) { ++ sd->rps_ipi_list = NULL; ++ ++ local_irq_enable(); ++ ++ /* Send pending IPI's to kick RPS processing on remote cpus. */ ++ while (remsd) { ++ struct softnet_data *next = remsd->rps_ipi_next; ++ ++ if (cpu_online(remsd->cpu)) ++ smp_call_function_single_async(remsd->cpu, ++ &remsd->csd); ++ remsd = next; ++ } ++ } else ++#endif ++ local_irq_enable(); ++} ++ ++static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ return sd->rps_ipi_list != NULL; ++#else ++ return false; ++#endif ++} ++ ++static int process_backlog(struct napi_struct *napi, int quota) ++{ ++ int work = 0; ++ struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); ++ ++ /* Check if we have pending ipi, its better to send them now, ++ * not waiting net_rx_action() end. ++ */ ++ if (sd_has_rps_ipi_waiting(sd)) { ++ local_irq_disable(); ++ net_rps_action_and_irq_enable(sd); ++ } ++ ++ napi->weight = weight_p; ++ local_irq_disable(); ++ while (1) { ++ struct sk_buff *skb; ++ ++ while ((skb = __skb_dequeue(&sd->process_queue))) { ++ rcu_read_lock(); ++ local_irq_enable(); ++ __netif_receive_skb(skb); ++ rcu_read_unlock(); ++ local_irq_disable(); ++ input_queue_head_incr(sd); ++ if (++work >= quota) { ++ local_irq_enable(); ++ return work; ++ } ++ } ++ ++ rps_lock(sd); ++ if (skb_queue_empty(&sd->input_pkt_queue)) { ++ /* ++ * Inline a custom version of __napi_complete(). ++ * only current cpu owns and manipulates this napi, ++ * and NAPI_STATE_SCHED is the only possible flag set ++ * on backlog. ++ * We can use a plain write instead of clear_bit(), ++ * and we dont need an smp_mb() memory barrier. ++ */ ++ napi->state = 0; ++ rps_unlock(sd); ++ ++ break; ++ } ++ ++ skb_queue_splice_tail_init(&sd->input_pkt_queue, ++ &sd->process_queue); ++ rps_unlock(sd); ++ } ++ local_irq_enable(); ++ ++ return work; ++} ++ ++/** ++ * __napi_schedule - schedule for receive ++ * @n: entry to schedule ++ * ++ * The entry's receive function will be scheduled to run. ++ * Consider using __napi_schedule_irqoff() if hard irqs are masked. ++ */ ++void __napi_schedule(struct napi_struct *n) ++{ ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ ____napi_schedule(this_cpu_ptr(&softnet_data), n); ++ local_irq_restore(flags); ++} ++EXPORT_SYMBOL(__napi_schedule); ++ ++/** ++ * __napi_schedule_irqoff - schedule for receive ++ * @n: entry to schedule ++ * ++ * Variant of __napi_schedule() assuming hard irqs are masked ++ */ ++void __napi_schedule_irqoff(struct napi_struct *n) ++{ ++ ____napi_schedule(this_cpu_ptr(&softnet_data), n); ++} ++EXPORT_SYMBOL(__napi_schedule_irqoff); ++ ++void __napi_complete(struct napi_struct *n) ++{ ++ BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); ++ ++ list_del_init(&n->poll_list); ++ smp_mb__before_atomic(); ++ clear_bit(NAPI_STATE_SCHED, &n->state); ++} ++EXPORT_SYMBOL(__napi_complete); ++ ++void napi_complete_done(struct napi_struct *n, int work_done) ++{ ++ unsigned long flags; ++ ++ /* ++ * don't let napi dequeue from the cpu poll list ++ * just in case its running on a different cpu ++ */ ++ if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state))) ++ return; ++ ++ if (n->gro_list) { ++ unsigned long timeout = 0; ++ ++ if (work_done) ++ timeout = n->dev->gro_flush_timeout; ++ ++ if (timeout) ++ hrtimer_start(&n->timer, ns_to_ktime(timeout), ++ HRTIMER_MODE_REL_PINNED); ++ else ++ napi_gro_flush(n, false); ++ } ++ if (likely(list_empty(&n->poll_list))) { ++ WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state)); ++ } else { ++ /* If n->poll_list is not empty, we need to mask irqs */ ++ local_irq_save(flags); ++ __napi_complete(n); ++ local_irq_restore(flags); ++ } ++} ++EXPORT_SYMBOL(napi_complete_done); ++ ++/* must be called under rcu_read_lock(), as we dont take a reference */ ++struct napi_struct *napi_by_id(unsigned int napi_id) ++{ ++ unsigned int hash = napi_id % HASH_SIZE(napi_hash); ++ struct napi_struct *napi; ++ ++ hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) ++ if (napi->napi_id == napi_id) ++ return napi; ++ ++ return NULL; ++} ++EXPORT_SYMBOL_GPL(napi_by_id); ++ ++void napi_hash_add(struct napi_struct *napi) ++{ ++ if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) { ++ ++ spin_lock(&napi_hash_lock); ++ ++ /* 0 is not a valid id, we also skip an id that is taken ++ * we expect both events to be extremely rare ++ */ ++ napi->napi_id = 0; ++ while (!napi->napi_id) { ++ napi->napi_id = ++napi_gen_id; ++ if (napi_by_id(napi->napi_id)) ++ napi->napi_id = 0; ++ } ++ ++ hlist_add_head_rcu(&napi->napi_hash_node, ++ &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); ++ ++ spin_unlock(&napi_hash_lock); ++ } ++} ++EXPORT_SYMBOL_GPL(napi_hash_add); ++ ++/* Warning : caller is responsible to make sure rcu grace period ++ * is respected before freeing memory containing @napi ++ */ ++void napi_hash_del(struct napi_struct *napi) ++{ ++ spin_lock(&napi_hash_lock); ++ ++ if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) ++ hlist_del_rcu(&napi->napi_hash_node); ++ ++ spin_unlock(&napi_hash_lock); ++} ++EXPORT_SYMBOL_GPL(napi_hash_del); ++ ++static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) ++{ ++ struct napi_struct *napi; ++ ++ napi = container_of(timer, struct napi_struct, timer); ++ if (napi->gro_list) ++ napi_schedule(napi); ++ ++ return HRTIMER_NORESTART; ++} ++ ++void netif_napi_add(struct net_device *dev, struct napi_struct *napi, ++ int (*poll)(struct napi_struct *, int), int weight) ++{ ++ INIT_LIST_HEAD(&napi->poll_list); ++ hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); ++ napi->timer.function = napi_watchdog; ++ napi->gro_count = 0; ++ napi->gro_list = NULL; ++ napi->skb = NULL; ++ napi->poll = poll; ++ if (weight > NAPI_POLL_WEIGHT) ++ pr_err_once("netif_napi_add() called with weight %d on device %s\n", ++ weight, dev->name); ++ napi->weight = weight; ++ list_add(&napi->dev_list, &dev->napi_list); ++ napi->dev = dev; ++#ifdef CONFIG_NETPOLL ++ spin_lock_init(&napi->poll_lock); ++ napi->poll_owner = -1; ++#endif ++ set_bit(NAPI_STATE_SCHED, &napi->state); ++} ++EXPORT_SYMBOL(netif_napi_add); ++ ++void napi_disable(struct napi_struct *n) ++{ ++ might_sleep(); ++ set_bit(NAPI_STATE_DISABLE, &n->state); ++ ++ while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) ++ msleep(1); ++ ++ hrtimer_cancel(&n->timer); ++ ++ clear_bit(NAPI_STATE_DISABLE, &n->state); ++} ++EXPORT_SYMBOL(napi_disable); ++ ++void netif_napi_del(struct napi_struct *napi) ++{ ++ list_del_init(&napi->dev_list); ++ napi_free_frags(napi); ++ ++ kfree_skb_list(napi->gro_list); ++ napi->gro_list = NULL; ++ napi->gro_count = 0; ++} ++EXPORT_SYMBOL(netif_napi_del); ++ ++static int napi_poll(struct napi_struct *n, struct list_head *repoll) ++{ ++ void *have; ++ int work, weight; ++ ++ list_del_init(&n->poll_list); ++ ++ have = netpoll_poll_lock(n); ++ ++ weight = n->weight; ++ ++ /* This NAPI_STATE_SCHED test is for avoiding a race ++ * with netpoll's poll_napi(). Only the entity which ++ * obtains the lock and sees NAPI_STATE_SCHED set will ++ * actually make the ->poll() call. Therefore we avoid ++ * accidentally calling ->poll() when NAPI is not scheduled. ++ */ ++ work = 0; ++ if (test_bit(NAPI_STATE_SCHED, &n->state)) { ++ work = n->poll(n, weight); ++ trace_napi_poll(n); ++ } ++ ++ WARN_ON_ONCE(work > weight); ++ ++ if (likely(work < weight)) ++ goto out_unlock; ++ ++ /* Drivers must not modify the NAPI state if they ++ * consume the entire weight. In such cases this code ++ * still "owns" the NAPI instance and therefore can ++ * move the instance around on the list at-will. ++ */ ++ if (unlikely(napi_disable_pending(n))) { ++ napi_complete(n); ++ goto out_unlock; ++ } ++ ++ if (n->gro_list) { ++ /* flush too old packets ++ * If HZ < 1000, flush all packets. ++ */ ++ napi_gro_flush(n, HZ >= 1000); ++ } ++ ++ /* Some drivers may have called napi_schedule ++ * prior to exhausting their budget. ++ */ ++ if (unlikely(!list_empty(&n->poll_list))) { ++ pr_warn_once("%s: Budget exhausted after napi rescheduled\n", ++ n->dev ? n->dev->name : "backlog"); ++ goto out_unlock; ++ } ++ ++ list_add_tail(&n->poll_list, repoll); ++ ++out_unlock: ++ netpoll_poll_unlock(have); ++ ++ return work; ++} ++ ++static void net_rx_action(struct softirq_action *h) ++{ ++ struct softnet_data *sd = this_cpu_ptr(&softnet_data); ++ unsigned long time_limit = jiffies + 2; ++ int budget = netdev_budget; ++ LIST_HEAD(list); ++ LIST_HEAD(repoll); ++ ++ local_irq_disable(); ++ list_splice_init(&sd->poll_list, &list); ++ local_irq_enable(); ++ ++ for (;;) { ++ struct napi_struct *n; ++ ++ if (list_empty(&list)) { ++ if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) ++ return; ++ break; ++ } ++ ++ n = list_first_entry(&list, struct napi_struct, poll_list); ++ budget -= napi_poll(n, &repoll); ++ ++ /* If softirq window is exhausted then punt. ++ * Allow this to run for 2 jiffies since which will allow ++ * an average latency of 1.5/HZ. ++ */ ++ if (unlikely(budget <= 0 || ++ time_after_eq(jiffies, time_limit))) { ++ sd->time_squeeze++; ++ break; ++ } ++ } ++ ++ local_irq_disable(); ++ ++ list_splice_tail_init(&sd->poll_list, &list); ++ list_splice_tail(&repoll, &list); ++ list_splice(&list, &sd->poll_list); ++ if (!list_empty(&sd->poll_list)) ++ __raise_softirq_irqoff(NET_RX_SOFTIRQ); ++ ++ net_rps_action_and_irq_enable(sd); ++} ++ ++struct netdev_adjacent { ++ struct net_device *dev; ++ ++ /* upper master flag, there can only be one master device per list */ ++ bool master; ++ ++ /* counter for the number of times this device was added to us */ ++ u16 ref_nr; ++ ++ /* private field for the users */ ++ void *private; ++ ++ struct list_head list; ++ struct rcu_head rcu; ++}; ++ ++static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *adj_list) ++{ ++ struct netdev_adjacent *adj; ++ ++ list_for_each_entry(adj, adj_list, list) { ++ if (adj->dev == adj_dev) ++ return adj; ++ } ++ return NULL; ++} ++ ++/** ++ * netdev_has_upper_dev - Check if device is linked to an upper device ++ * @dev: device ++ * @upper_dev: upper device to check ++ * ++ * Find out if a device is linked to specified upper device and return true ++ * in case it is. Note that this checks only immediate upper device, ++ * not through a complete stack of devices. The caller must hold the RTNL lock. ++ */ ++bool netdev_has_upper_dev(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ ASSERT_RTNL(); ++ ++ return __netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper); ++} ++EXPORT_SYMBOL(netdev_has_upper_dev); ++ ++/** ++ * netdev_has_any_upper_dev - Check if device is linked to some device ++ * @dev: device ++ * ++ * Find out if a device is linked to an upper device and return true in case ++ * it is. The caller must hold the RTNL lock. ++ */ ++static bool netdev_has_any_upper_dev(struct net_device *dev) ++{ ++ ASSERT_RTNL(); ++ ++ return !list_empty(&dev->all_adj_list.upper); ++} ++ ++/** ++ * netdev_master_upper_dev_get - Get master upper device ++ * @dev: device ++ * ++ * Find a master upper device and return pointer to it or NULL in case ++ * it's not there. The caller must hold the RTNL lock. ++ */ ++struct net_device *netdev_master_upper_dev_get(struct net_device *dev) ++{ ++ struct netdev_adjacent *upper; ++ ++ ASSERT_RTNL(); ++ ++ if (list_empty(&dev->adj_list.upper)) ++ return NULL; ++ ++ upper = list_first_entry(&dev->adj_list.upper, ++ struct netdev_adjacent, list); ++ if (likely(upper->master)) ++ return upper->dev; ++ return NULL; ++} ++EXPORT_SYMBOL(netdev_master_upper_dev_get); ++ ++void *netdev_adjacent_get_private(struct list_head *adj_list) ++{ ++ struct netdev_adjacent *adj; ++ ++ adj = list_entry(adj_list, struct netdev_adjacent, list); ++ ++ return adj->private; ++} ++EXPORT_SYMBOL(netdev_adjacent_get_private); ++ ++/** ++ * netdev_upper_get_next_dev_rcu - Get the next dev from upper list ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next device from the dev's upper list, starting from iter ++ * position. The caller must hold RCU read lock. ++ */ ++struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, ++ struct list_head **iter) ++{ ++ struct netdev_adjacent *upper; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); ++ ++ upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); ++ ++ if (&upper->list == &dev->adj_list.upper) ++ return NULL; ++ ++ *iter = &upper->list; ++ ++ return upper->dev; ++} ++EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); ++ ++/** ++ * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next device from the dev's upper list, starting from iter ++ * position. The caller must hold RCU read lock. ++ */ ++struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev, ++ struct list_head **iter) ++{ ++ struct netdev_adjacent *upper; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); ++ ++ upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); ++ ++ if (&upper->list == &dev->all_adj_list.upper) ++ return NULL; ++ ++ *iter = &upper->list; ++ ++ return upper->dev; ++} ++EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu); ++ ++/** ++ * netdev_lower_get_next_private - Get the next ->private from the ++ * lower neighbour list ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next netdev_adjacent->private from the dev's lower neighbour ++ * list, starting from iter position. The caller must hold either hold the ++ * RTNL lock or its own locking that guarantees that the neighbour lower ++ * list will remain unchainged. ++ */ ++void *netdev_lower_get_next_private(struct net_device *dev, ++ struct list_head **iter) ++{ ++ struct netdev_adjacent *lower; ++ ++ lower = list_entry(*iter, struct netdev_adjacent, list); ++ ++ if (&lower->list == &dev->adj_list.lower) ++ return NULL; ++ ++ *iter = lower->list.next; ++ ++ return lower->private; ++} ++EXPORT_SYMBOL(netdev_lower_get_next_private); ++ ++/** ++ * netdev_lower_get_next_private_rcu - Get the next ->private from the ++ * lower neighbour list, RCU ++ * variant ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next netdev_adjacent->private from the dev's lower neighbour ++ * list, starting from iter position. The caller must hold RCU read lock. ++ */ ++void *netdev_lower_get_next_private_rcu(struct net_device *dev, ++ struct list_head **iter) ++{ ++ struct netdev_adjacent *lower; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held()); ++ ++ lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); ++ ++ if (&lower->list == &dev->adj_list.lower) ++ return NULL; ++ ++ *iter = &lower->list; ++ ++ return lower->private; ++} ++EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); ++ ++/** ++ * netdev_lower_get_next - Get the next device from the lower neighbour ++ * list ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next netdev_adjacent from the dev's lower neighbour ++ * list, starting from iter position. The caller must hold RTNL lock or ++ * its own locking that guarantees that the neighbour lower ++ * list will remain unchainged. ++ */ ++void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) ++{ ++ struct netdev_adjacent *lower; ++ ++ lower = list_entry((*iter)->next, struct netdev_adjacent, list); ++ ++ if (&lower->list == &dev->adj_list.lower) ++ return NULL; ++ ++ *iter = &lower->list; ++ ++ return lower->dev; ++} ++EXPORT_SYMBOL(netdev_lower_get_next); ++ ++/** ++ * netdev_lower_get_first_private_rcu - Get the first ->private from the ++ * lower neighbour list, RCU ++ * variant ++ * @dev: device ++ * ++ * Gets the first netdev_adjacent->private from the dev's lower neighbour ++ * list. The caller must hold RCU read lock. ++ */ ++void *netdev_lower_get_first_private_rcu(struct net_device *dev) ++{ ++ struct netdev_adjacent *lower; ++ ++ lower = list_first_or_null_rcu(&dev->adj_list.lower, ++ struct netdev_adjacent, list); ++ if (lower) ++ return lower->private; ++ return NULL; ++} ++EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); ++ ++/** ++ * netdev_master_upper_dev_get_rcu - Get master upper device ++ * @dev: device ++ * ++ * Find a master upper device and return pointer to it or NULL in case ++ * it's not there. The caller must hold the RCU read lock. ++ */ ++struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) ++{ ++ struct netdev_adjacent *upper; ++ ++ upper = list_first_or_null_rcu(&dev->adj_list.upper, ++ struct netdev_adjacent, list); ++ if (upper && likely(upper->master)) ++ return upper->dev; ++ return NULL; ++} ++EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); ++ ++static int netdev_adjacent_sysfs_add(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *dev_list) ++{ ++ char linkname[IFNAMSIZ+7]; ++ sprintf(linkname, dev_list == &dev->adj_list.upper ? ++ "upper_%s" : "lower_%s", adj_dev->name); ++ return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), ++ linkname); ++} ++static void netdev_adjacent_sysfs_del(struct net_device *dev, ++ char *name, ++ struct list_head *dev_list) ++{ ++ char linkname[IFNAMSIZ+7]; ++ sprintf(linkname, dev_list == &dev->adj_list.upper ? ++ "upper_%s" : "lower_%s", name); ++ sysfs_remove_link(&(dev->dev.kobj), linkname); ++} ++ ++static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *dev_list) ++{ ++ return (dev_list == &dev->adj_list.upper || ++ dev_list == &dev->adj_list.lower) && ++ net_eq(dev_net(dev), dev_net(adj_dev)); ++} ++ ++static int __netdev_adjacent_dev_insert(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *dev_list, ++ void *private, bool master) ++{ ++ struct netdev_adjacent *adj; ++ int ret; ++ ++ adj = __netdev_find_adj(dev, adj_dev, dev_list); ++ ++ if (adj) { ++ adj->ref_nr++; ++ return 0; ++ } ++ ++ adj = kmalloc(sizeof(*adj), GFP_KERNEL); ++ if (!adj) ++ return -ENOMEM; ++ ++ adj->dev = adj_dev; ++ adj->master = master; ++ adj->ref_nr = 1; ++ adj->private = private; ++ dev_hold(adj_dev); ++ ++ pr_debug("dev_hold for %s, because of link added from %s to %s\n", ++ adj_dev->name, dev->name, adj_dev->name); ++ ++ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { ++ ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); ++ if (ret) ++ goto free_adj; ++ } ++ ++ /* Ensure that master link is always the first item in list. */ ++ if (master) { ++ ret = sysfs_create_link(&(dev->dev.kobj), ++ &(adj_dev->dev.kobj), "master"); ++ if (ret) ++ goto remove_symlinks; ++ ++ list_add_rcu(&adj->list, dev_list); ++ } else { ++ list_add_tail_rcu(&adj->list, dev_list); ++ } ++ ++ return 0; ++ ++remove_symlinks: ++ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) ++ netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); ++free_adj: ++ kfree(adj); ++ dev_put(adj_dev); ++ ++ return ret; ++} ++ ++static void __netdev_adjacent_dev_remove(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *dev_list) ++{ ++ struct netdev_adjacent *adj; ++ ++ adj = __netdev_find_adj(dev, adj_dev, dev_list); ++ ++ if (!adj) { ++ pr_err("tried to remove device %s from %s\n", ++ dev->name, adj_dev->name); ++ BUG(); ++ } ++ ++ if (adj->ref_nr > 1) { ++ pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name, ++ adj->ref_nr-1); ++ adj->ref_nr--; ++ return; ++ } ++ ++ if (adj->master) ++ sysfs_remove_link(&(dev->dev.kobj), "master"); ++ ++ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) ++ netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); ++ ++ list_del_rcu(&adj->list); ++ pr_debug("dev_put for %s, because link removed from %s to %s\n", ++ adj_dev->name, dev->name, adj_dev->name); ++ dev_put(adj_dev); ++ kfree_rcu(adj, rcu); ++} ++ ++static int __netdev_adjacent_dev_link_lists(struct net_device *dev, ++ struct net_device *upper_dev, ++ struct list_head *up_list, ++ struct list_head *down_list, ++ void *private, bool master) ++{ ++ int ret; ++ ++ ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private, ++ master); ++ if (ret) ++ return ret; ++ ++ ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private, ++ false); ++ if (ret) { ++ __netdev_adjacent_dev_remove(dev, upper_dev, up_list); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int __netdev_adjacent_dev_link(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ return __netdev_adjacent_dev_link_lists(dev, upper_dev, ++ &dev->all_adj_list.upper, ++ &upper_dev->all_adj_list.lower, ++ NULL, false); ++} ++ ++static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, ++ struct net_device *upper_dev, ++ struct list_head *up_list, ++ struct list_head *down_list) ++{ ++ __netdev_adjacent_dev_remove(dev, upper_dev, up_list); ++ __netdev_adjacent_dev_remove(upper_dev, dev, down_list); ++} ++ ++static void __netdev_adjacent_dev_unlink(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ __netdev_adjacent_dev_unlink_lists(dev, upper_dev, ++ &dev->all_adj_list.upper, ++ &upper_dev->all_adj_list.lower); ++} ++ ++static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, ++ struct net_device *upper_dev, ++ void *private, bool master) ++{ ++ int ret = __netdev_adjacent_dev_link(dev, upper_dev); ++ ++ if (ret) ++ return ret; ++ ++ ret = __netdev_adjacent_dev_link_lists(dev, upper_dev, ++ &dev->adj_list.upper, ++ &upper_dev->adj_list.lower, ++ private, master); ++ if (ret) { ++ __netdev_adjacent_dev_unlink(dev, upper_dev); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ __netdev_adjacent_dev_unlink(dev, upper_dev); ++ __netdev_adjacent_dev_unlink_lists(dev, upper_dev, ++ &dev->adj_list.upper, ++ &upper_dev->adj_list.lower); ++} ++ ++static int __netdev_upper_dev_link(struct net_device *dev, ++ struct net_device *upper_dev, bool master, ++ void *private) ++{ ++ struct netdev_adjacent *i, *j, *to_i, *to_j; ++ int ret = 0; ++ ++ ASSERT_RTNL(); ++ ++ if (dev == upper_dev) ++ return -EBUSY; ++ ++ /* To prevent loops, check if dev is not upper device to upper_dev. */ ++ if (__netdev_find_adj(upper_dev, dev, &upper_dev->all_adj_list.upper)) ++ return -EBUSY; ++ ++ if (__netdev_find_adj(dev, upper_dev, &dev->adj_list.upper)) ++ return -EEXIST; ++ ++ if (master && netdev_master_upper_dev_get(dev)) ++ return -EBUSY; ++ ++ ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private, ++ master); ++ if (ret) ++ return ret; ++ ++ /* Now that we linked these devs, make all the upper_dev's ++ * all_adj_list.upper visible to every dev's all_adj_list.lower an ++ * versa, and don't forget the devices itself. All of these ++ * links are non-neighbours. ++ */ ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) { ++ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { ++ pr_debug("Interlinking %s with %s, non-neighbour\n", ++ i->dev->name, j->dev->name); ++ ret = __netdev_adjacent_dev_link(i->dev, j->dev); ++ if (ret) ++ goto rollback_mesh; ++ } ++ } ++ ++ /* add dev to every upper_dev's upper device */ ++ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { ++ pr_debug("linking %s's upper device %s with %s\n", ++ upper_dev->name, i->dev->name, dev->name); ++ ret = __netdev_adjacent_dev_link(dev, i->dev); ++ if (ret) ++ goto rollback_upper_mesh; ++ } ++ ++ /* add upper_dev to every dev's lower device */ ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) { ++ pr_debug("linking %s's lower device %s with %s\n", dev->name, ++ i->dev->name, upper_dev->name); ++ ret = __netdev_adjacent_dev_link(i->dev, upper_dev); ++ if (ret) ++ goto rollback_lower_mesh; ++ } ++ ++ call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev); ++ return 0; ++ ++rollback_lower_mesh: ++ to_i = i; ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) { ++ if (i == to_i) ++ break; ++ __netdev_adjacent_dev_unlink(i->dev, upper_dev); ++ } ++ ++ i = NULL; ++ ++rollback_upper_mesh: ++ to_i = i; ++ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { ++ if (i == to_i) ++ break; ++ __netdev_adjacent_dev_unlink(dev, i->dev); ++ } ++ ++ i = j = NULL; ++ ++rollback_mesh: ++ to_i = i; ++ to_j = j; ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) { ++ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { ++ if (i == to_i && j == to_j) ++ break; ++ __netdev_adjacent_dev_unlink(i->dev, j->dev); ++ } ++ if (i == to_i) ++ break; ++ } ++ ++ __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); ++ ++ return ret; ++} ++ ++/** ++ * netdev_upper_dev_link - Add a link to the upper device ++ * @dev: device ++ * @upper_dev: new upper device ++ * ++ * Adds a link to device which is upper to this one. The caller must hold ++ * the RTNL lock. On a failure a negative errno code is returned. ++ * On success the reference counts are adjusted and the function ++ * returns zero. ++ */ ++int netdev_upper_dev_link(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ return __netdev_upper_dev_link(dev, upper_dev, false, NULL); ++} ++EXPORT_SYMBOL(netdev_upper_dev_link); ++ ++/** ++ * netdev_master_upper_dev_link - Add a master link to the upper device ++ * @dev: device ++ * @upper_dev: new upper device ++ * ++ * Adds a link to device which is upper to this one. In this case, only ++ * one master upper device can be linked, although other non-master devices ++ * might be linked as well. The caller must hold the RTNL lock. ++ * On a failure a negative errno code is returned. On success the reference ++ * counts are adjusted and the function returns zero. ++ */ ++int netdev_master_upper_dev_link(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ return __netdev_upper_dev_link(dev, upper_dev, true, NULL); ++} ++EXPORT_SYMBOL(netdev_master_upper_dev_link); ++ ++int netdev_master_upper_dev_link_private(struct net_device *dev, ++ struct net_device *upper_dev, ++ void *private) ++{ ++ return __netdev_upper_dev_link(dev, upper_dev, true, private); ++} ++EXPORT_SYMBOL(netdev_master_upper_dev_link_private); ++ ++/** ++ * netdev_upper_dev_unlink - Removes a link to upper device ++ * @dev: device ++ * @upper_dev: new upper device ++ * ++ * Removes a link to device which is upper to this one. The caller must hold ++ * the RTNL lock. ++ */ ++void netdev_upper_dev_unlink(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ struct netdev_adjacent *i, *j; ++ ASSERT_RTNL(); ++ ++ __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); ++ ++ /* Here is the tricky part. We must remove all dev's lower ++ * devices from all upper_dev's upper devices and vice ++ * versa, to maintain the graph relationship. ++ */ ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) ++ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) ++ __netdev_adjacent_dev_unlink(i->dev, j->dev); ++ ++ /* remove also the devices itself from lower/upper device ++ * list ++ */ ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) ++ __netdev_adjacent_dev_unlink(i->dev, upper_dev); ++ ++ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) ++ __netdev_adjacent_dev_unlink(dev, i->dev); ++ ++ call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev); ++} ++EXPORT_SYMBOL(netdev_upper_dev_unlink); ++ ++/** ++ * netdev_bonding_info_change - Dispatch event about slave change ++ * @dev: device ++ * @bonding_info: info to dispatch ++ * ++ * Send NETDEV_BONDING_INFO to netdev notifiers with info. ++ * The caller must hold the RTNL lock. ++ */ ++void netdev_bonding_info_change(struct net_device *dev, ++ struct netdev_bonding_info *bonding_info) ++{ ++ struct netdev_notifier_bonding_info info; ++ ++ memcpy(&info.bonding_info, bonding_info, ++ sizeof(struct netdev_bonding_info)); ++ call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev, ++ &info.info); ++} ++EXPORT_SYMBOL(netdev_bonding_info_change); ++ ++static void netdev_adjacent_add_links(struct net_device *dev) ++{ ++ struct netdev_adjacent *iter; ++ ++ struct net *net = dev_net(dev); ++ ++ list_for_each_entry(iter, &dev->adj_list.upper, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_add(iter->dev, dev, ++ &iter->dev->adj_list.lower); ++ netdev_adjacent_sysfs_add(dev, iter->dev, ++ &dev->adj_list.upper); ++ } ++ ++ list_for_each_entry(iter, &dev->adj_list.lower, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_add(iter->dev, dev, ++ &iter->dev->adj_list.upper); ++ netdev_adjacent_sysfs_add(dev, iter->dev, ++ &dev->adj_list.lower); ++ } ++} ++ ++static void netdev_adjacent_del_links(struct net_device *dev) ++{ ++ struct netdev_adjacent *iter; ++ ++ struct net *net = dev_net(dev); ++ ++ list_for_each_entry(iter, &dev->adj_list.upper, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_del(iter->dev, dev->name, ++ &iter->dev->adj_list.lower); ++ netdev_adjacent_sysfs_del(dev, iter->dev->name, ++ &dev->adj_list.upper); ++ } ++ ++ list_for_each_entry(iter, &dev->adj_list.lower, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_del(iter->dev, dev->name, ++ &iter->dev->adj_list.upper); ++ netdev_adjacent_sysfs_del(dev, iter->dev->name, ++ &dev->adj_list.lower); ++ } ++} ++ ++void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) ++{ ++ struct netdev_adjacent *iter; ++ ++ struct net *net = dev_net(dev); ++ ++ list_for_each_entry(iter, &dev->adj_list.upper, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_del(iter->dev, oldname, ++ &iter->dev->adj_list.lower); ++ netdev_adjacent_sysfs_add(iter->dev, dev, ++ &iter->dev->adj_list.lower); ++ } ++ ++ list_for_each_entry(iter, &dev->adj_list.lower, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_del(iter->dev, oldname, ++ &iter->dev->adj_list.upper); ++ netdev_adjacent_sysfs_add(iter->dev, dev, ++ &iter->dev->adj_list.upper); ++ } ++} ++ ++void *netdev_lower_dev_get_private(struct net_device *dev, ++ struct net_device *lower_dev) ++{ ++ struct netdev_adjacent *lower; ++ ++ if (!lower_dev) ++ return NULL; ++ lower = __netdev_find_adj(dev, lower_dev, &dev->adj_list.lower); ++ if (!lower) ++ return NULL; ++ ++ return lower->private; ++} ++EXPORT_SYMBOL(netdev_lower_dev_get_private); ++ ++ ++int dev_get_nest_level(struct net_device *dev, ++ bool (*type_check)(struct net_device *dev)) ++{ ++ struct net_device *lower = NULL; ++ struct list_head *iter; ++ int max_nest = -1; ++ int nest; ++ ++ ASSERT_RTNL(); ++ ++ netdev_for_each_lower_dev(dev, lower, iter) { ++ nest = dev_get_nest_level(lower, type_check); ++ if (max_nest < nest) ++ max_nest = nest; ++ } ++ ++ if (type_check(dev)) ++ max_nest++; ++ ++ return max_nest; ++} ++EXPORT_SYMBOL(dev_get_nest_level); ++ ++static void dev_change_rx_flags(struct net_device *dev, int flags) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (ops->ndo_change_rx_flags) ++ ops->ndo_change_rx_flags(dev, flags); ++} ++ ++static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) ++{ ++ unsigned int old_flags = dev->flags; ++ kuid_t uid; ++ kgid_t gid; ++ ++ ASSERT_RTNL(); ++ ++ dev->flags |= IFF_PROMISC; ++ dev->promiscuity += inc; ++ if (dev->promiscuity == 0) { ++ /* ++ * Avoid overflow. ++ * If inc causes overflow, untouch promisc and return error. ++ */ ++ if (inc < 0) ++ dev->flags &= ~IFF_PROMISC; ++ else { ++ dev->promiscuity -= inc; ++ pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", ++ dev->name); ++ return -EOVERFLOW; ++ } ++ } ++ if (dev->flags != old_flags) { ++ pr_info("device %s %s promiscuous mode\n", ++ dev->name, ++ dev->flags & IFF_PROMISC ? "entered" : "left"); ++ if (audit_enabled) { ++ current_uid_gid(&uid, &gid); ++ audit_log(current->audit_context, GFP_ATOMIC, ++ AUDIT_ANOM_PROMISCUOUS, ++ "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", ++ dev->name, (dev->flags & IFF_PROMISC), ++ (old_flags & IFF_PROMISC), ++ from_kuid(&init_user_ns, audit_get_loginuid(current)), ++ from_kuid(&init_user_ns, uid), ++ from_kgid(&init_user_ns, gid), ++ audit_get_sessionid(current)); ++ } ++ ++ dev_change_rx_flags(dev, IFF_PROMISC); ++ } ++ if (notify) ++ __dev_notify_flags(dev, old_flags, IFF_PROMISC); ++ return 0; ++} ++ ++/** ++ * dev_set_promiscuity - update promiscuity count on a device ++ * @dev: device ++ * @inc: modifier ++ * ++ * Add or remove promiscuity from a device. While the count in the device ++ * remains above zero the interface remains promiscuous. Once it hits zero ++ * the device reverts back to normal filtering operation. A negative inc ++ * value is used to drop promiscuity on the device. ++ * Return 0 if successful or a negative errno code on error. ++ */ ++int dev_set_promiscuity(struct net_device *dev, int inc) ++{ ++ unsigned int old_flags = dev->flags; ++ int err; ++ ++ err = __dev_set_promiscuity(dev, inc, true); ++ if (err < 0) ++ return err; ++ if (dev->flags != old_flags) ++ dev_set_rx_mode(dev); ++ return err; ++} ++EXPORT_SYMBOL(dev_set_promiscuity); ++ ++static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) ++{ ++ unsigned int old_flags = dev->flags, old_gflags = dev->gflags; ++ ++ ASSERT_RTNL(); ++ ++ dev->flags |= IFF_ALLMULTI; ++ dev->allmulti += inc; ++ if (dev->allmulti == 0) { ++ /* ++ * Avoid overflow. ++ * If inc causes overflow, untouch allmulti and return error. ++ */ ++ if (inc < 0) ++ dev->flags &= ~IFF_ALLMULTI; ++ else { ++ dev->allmulti -= inc; ++ pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", ++ dev->name); ++ return -EOVERFLOW; ++ } ++ } ++ if (dev->flags ^ old_flags) { ++ dev_change_rx_flags(dev, IFF_ALLMULTI); ++ dev_set_rx_mode(dev); ++ if (notify) ++ __dev_notify_flags(dev, old_flags, ++ dev->gflags ^ old_gflags); ++ } ++ return 0; ++} ++ ++/** ++ * dev_set_allmulti - update allmulti count on a device ++ * @dev: device ++ * @inc: modifier ++ * ++ * Add or remove reception of all multicast frames to a device. While the ++ * count in the device remains above zero the interface remains listening ++ * to all interfaces. Once it hits zero the device reverts back to normal ++ * filtering operation. A negative @inc value is used to drop the counter ++ * when releasing a resource needing all multicasts. ++ * Return 0 if successful or a negative errno code on error. ++ */ ++ ++int dev_set_allmulti(struct net_device *dev, int inc) ++{ ++ return __dev_set_allmulti(dev, inc, true); ++} ++EXPORT_SYMBOL(dev_set_allmulti); ++ ++/* ++ * Upload unicast and multicast address lists to device and ++ * configure RX filtering. When the device doesn't support unicast ++ * filtering it is put in promiscuous mode while unicast addresses ++ * are present. ++ */ ++void __dev_set_rx_mode(struct net_device *dev) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ /* dev_open will call this function so the list will stay sane. */ ++ if (!(dev->flags&IFF_UP)) ++ return; ++ ++ if (!netif_device_present(dev)) ++ return; ++ ++ if (!(dev->priv_flags & IFF_UNICAST_FLT)) { ++ /* Unicast addresses changes may only happen under the rtnl, ++ * therefore calling __dev_set_promiscuity here is safe. ++ */ ++ if (!netdev_uc_empty(dev) && !dev->uc_promisc) { ++ __dev_set_promiscuity(dev, 1, false); ++ dev->uc_promisc = true; ++ } else if (netdev_uc_empty(dev) && dev->uc_promisc) { ++ __dev_set_promiscuity(dev, -1, false); ++ dev->uc_promisc = false; ++ } ++ } ++ ++ if (ops->ndo_set_rx_mode) ++ ops->ndo_set_rx_mode(dev); ++} ++ ++void dev_set_rx_mode(struct net_device *dev) ++{ ++ netif_addr_lock_bh(dev); ++ __dev_set_rx_mode(dev); ++ netif_addr_unlock_bh(dev); ++} ++ ++/** ++ * dev_get_flags - get flags reported to userspace ++ * @dev: device ++ * ++ * Get the combination of flag bits exported through APIs to userspace. ++ */ ++unsigned int dev_get_flags(const struct net_device *dev) ++{ ++ unsigned int flags; ++ ++ flags = (dev->flags & ~(IFF_PROMISC | ++ IFF_ALLMULTI | ++ IFF_RUNNING | ++ IFF_LOWER_UP | ++ IFF_DORMANT)) | ++ (dev->gflags & (IFF_PROMISC | ++ IFF_ALLMULTI)); ++ ++ if (netif_running(dev)) { ++ if (netif_oper_up(dev)) ++ flags |= IFF_RUNNING; ++ if (netif_carrier_ok(dev)) ++ flags |= IFF_LOWER_UP; ++ if (netif_dormant(dev)) ++ flags |= IFF_DORMANT; ++ } ++ ++ return flags; ++} ++EXPORT_SYMBOL(dev_get_flags); ++ ++int __dev_change_flags(struct net_device *dev, unsigned int flags) ++{ ++ unsigned int old_flags = dev->flags; ++ int ret; ++ ++ ASSERT_RTNL(); ++ ++ /* ++ * Set the flags on our device. ++ */ ++ ++ dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | ++ IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | ++ IFF_AUTOMEDIA)) | ++ (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | ++ IFF_ALLMULTI)); ++ ++ /* ++ * Load in the correct multicast list now the flags have changed. ++ */ ++ ++ if ((old_flags ^ flags) & IFF_MULTICAST) ++ dev_change_rx_flags(dev, IFF_MULTICAST); ++ ++ dev_set_rx_mode(dev); ++ ++ /* ++ * Have we downed the interface. We handle IFF_UP ourselves ++ * according to user attempts to set it, rather than blindly ++ * setting it. ++ */ ++ ++ ret = 0; ++ if ((old_flags ^ flags) & IFF_UP) ++ ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev); ++ ++ if ((flags ^ dev->gflags) & IFF_PROMISC) { ++ int inc = (flags & IFF_PROMISC) ? 1 : -1; ++ unsigned int old_flags = dev->flags; ++ ++ dev->gflags ^= IFF_PROMISC; ++ ++ if (__dev_set_promiscuity(dev, inc, false) >= 0) ++ if (dev->flags != old_flags) ++ dev_set_rx_mode(dev); ++ } ++ ++ /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI ++ is important. Some (broken) drivers set IFF_PROMISC, when ++ IFF_ALLMULTI is requested not asking us and not reporting. ++ */ ++ if ((flags ^ dev->gflags) & IFF_ALLMULTI) { ++ int inc = (flags & IFF_ALLMULTI) ? 1 : -1; ++ ++ dev->gflags ^= IFF_ALLMULTI; ++ __dev_set_allmulti(dev, inc, false); ++ } ++ ++ return ret; ++} ++ ++void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, ++ unsigned int gchanges) ++{ ++ unsigned int changes = dev->flags ^ old_flags; ++ ++ if (gchanges) ++ rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); ++ ++ if (changes & IFF_UP) { ++ if (dev->flags & IFF_UP) ++ call_netdevice_notifiers(NETDEV_UP, dev); ++ else ++ call_netdevice_notifiers(NETDEV_DOWN, dev); ++ } ++ ++ if (dev->flags & IFF_UP && ++ (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { ++ struct netdev_notifier_change_info change_info; ++ ++ change_info.flags_changed = changes; ++ call_netdevice_notifiers_info(NETDEV_CHANGE, dev, ++ &change_info.info); ++ } ++} ++ ++/** ++ * dev_change_flags - change device settings ++ * @dev: device ++ * @flags: device state flags ++ * ++ * Change settings on device based state flags. The flags are ++ * in the userspace exported format. ++ */ ++int dev_change_flags(struct net_device *dev, unsigned int flags) ++{ ++ int ret; ++ unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; ++ ++ ret = __dev_change_flags(dev, flags); ++ if (ret < 0) ++ return ret; ++ ++ changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); ++ __dev_notify_flags(dev, old_flags, changes); ++ return ret; ++} ++EXPORT_SYMBOL(dev_change_flags); ++ ++static int __dev_set_mtu(struct net_device *dev, int new_mtu) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (ops->ndo_change_mtu) ++ return ops->ndo_change_mtu(dev, new_mtu); ++ ++ dev->mtu = new_mtu; ++ return 0; ++} ++ ++/** ++ * dev_set_mtu - Change maximum transfer unit ++ * @dev: device ++ * @new_mtu: new transfer unit ++ * ++ * Change the maximum transfer size of the network device. ++ */ ++int dev_set_mtu(struct net_device *dev, int new_mtu) ++{ ++ int err, orig_mtu; ++ ++ if (new_mtu == dev->mtu) ++ return 0; ++ ++ /* MTU must be positive. */ ++ if (new_mtu < 0) ++ return -EINVAL; ++ ++ if (!netif_device_present(dev)) ++ return -ENODEV; ++ ++ err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); ++ err = notifier_to_errno(err); ++ if (err) ++ return err; ++ ++ orig_mtu = dev->mtu; ++ err = __dev_set_mtu(dev, new_mtu); ++ ++ if (!err) { ++ err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); ++ err = notifier_to_errno(err); ++ if (err) { ++ /* setting mtu back and notifying everyone again, ++ * so that they have a chance to revert changes. ++ */ ++ __dev_set_mtu(dev, orig_mtu); ++ call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); ++ } ++ } ++ return err; ++} ++EXPORT_SYMBOL(dev_set_mtu); ++ ++/** ++ * dev_set_group - Change group this device belongs to ++ * @dev: device ++ * @new_group: group this device should belong to ++ */ ++void dev_set_group(struct net_device *dev, int new_group) ++{ ++ dev->group = new_group; ++} ++EXPORT_SYMBOL(dev_set_group); ++ ++/** ++ * dev_set_mac_address - Change Media Access Control Address ++ * @dev: device ++ * @sa: new address ++ * ++ * Change the hardware (MAC) address of the device ++ */ ++int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ int err; ++ ++ if (!ops->ndo_set_mac_address) ++ return -EOPNOTSUPP; ++ if (sa->sa_family != dev->type) ++ return -EINVAL; ++ if (!netif_device_present(dev)) ++ return -ENODEV; ++ err = ops->ndo_set_mac_address(dev, sa); ++ if (err) ++ return err; ++ dev->addr_assign_type = NET_ADDR_SET; ++ call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); ++ add_device_randomness(dev->dev_addr, dev->addr_len); ++ return 0; ++} ++EXPORT_SYMBOL(dev_set_mac_address); ++ ++/** ++ * dev_change_carrier - Change device carrier ++ * @dev: device ++ * @new_carrier: new value ++ * ++ * Change device carrier ++ */ ++int dev_change_carrier(struct net_device *dev, bool new_carrier) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (!ops->ndo_change_carrier) ++ return -EOPNOTSUPP; ++ if (!netif_device_present(dev)) ++ return -ENODEV; ++ return ops->ndo_change_carrier(dev, new_carrier); ++} ++EXPORT_SYMBOL(dev_change_carrier); ++ ++/** ++ * dev_get_phys_port_id - Get device physical port ID ++ * @dev: device ++ * @ppid: port ID ++ * ++ * Get device physical port ID ++ */ ++int dev_get_phys_port_id(struct net_device *dev, ++ struct netdev_phys_item_id *ppid) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (!ops->ndo_get_phys_port_id) ++ return -EOPNOTSUPP; ++ return ops->ndo_get_phys_port_id(dev, ppid); ++} ++EXPORT_SYMBOL(dev_get_phys_port_id); ++ ++/** ++ * dev_get_phys_port_name - Get device physical port name ++ * @dev: device ++ * @name: port name ++ * ++ * Get device physical port name ++ */ ++int dev_get_phys_port_name(struct net_device *dev, ++ char *name, size_t len) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (!ops->ndo_get_phys_port_name) ++ return -EOPNOTSUPP; ++ return ops->ndo_get_phys_port_name(dev, name, len); ++} ++EXPORT_SYMBOL(dev_get_phys_port_name); ++ ++/** ++ * dev_new_index - allocate an ifindex ++ * @net: the applicable net namespace ++ * ++ * Returns a suitable unique value for a new device interface ++ * number. The caller must hold the rtnl semaphore or the ++ * dev_base_lock to be sure it remains unique. ++ */ ++static int dev_new_index(struct net *net) ++{ ++ int ifindex = net->ifindex; ++ for (;;) { ++ if (++ifindex <= 0) ++ ifindex = 1; ++ if (!__dev_get_by_index(net, ifindex)) ++ return net->ifindex = ifindex; ++ } ++} ++ ++/* Delayed registration/unregisteration */ ++static LIST_HEAD(net_todo_list); ++DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); ++ ++static void net_set_todo(struct net_device *dev) ++{ ++ list_add_tail(&dev->todo_list, &net_todo_list); ++ dev_net(dev)->dev_unreg_count++; ++} ++ ++static void rollback_registered_many(struct list_head *head) ++{ ++ struct net_device *dev, *tmp; ++ LIST_HEAD(close_head); ++ ++ BUG_ON(dev_boot_phase); ++ ASSERT_RTNL(); ++ ++ list_for_each_entry_safe(dev, tmp, head, unreg_list) { ++ /* Some devices call without registering ++ * for initialization unwind. Remove those ++ * devices and proceed with the remaining. ++ */ ++ if (dev->reg_state == NETREG_UNINITIALIZED) { ++ pr_debug("unregister_netdevice: device %s/%p never was registered\n", ++ dev->name, dev); ++ ++ WARN_ON(1); ++ list_del(&dev->unreg_list); ++ continue; ++ } ++ dev->dismantle = true; ++ BUG_ON(dev->reg_state != NETREG_REGISTERED); ++ } ++ ++ /* If device is running, close it first. */ ++ list_for_each_entry(dev, head, unreg_list) ++ list_add_tail(&dev->close_list, &close_head); ++ dev_close_many(&close_head, true); ++ ++ list_for_each_entry(dev, head, unreg_list) { ++ /* And unlink it from device chain. */ ++ unlist_netdevice(dev); ++ ++ dev->reg_state = NETREG_UNREGISTERING; ++ on_each_cpu(flush_backlog, dev, 1); ++ } ++ ++ synchronize_net(); ++ ++ list_for_each_entry(dev, head, unreg_list) { ++ struct sk_buff *skb = NULL; ++ ++ /* Shutdown queueing discipline. */ ++ dev_shutdown(dev); ++ ++ ++ /* Notify protocols, that we are about to destroy ++ this device. They should clean all the things. ++ */ ++ call_netdevice_notifiers(NETDEV_UNREGISTER, dev); ++ ++ if (!dev->rtnl_link_ops || ++ dev->rtnl_link_state == RTNL_LINK_INITIALIZED) ++ skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, ++ GFP_KERNEL); ++ ++ /* ++ * Flush the unicast and multicast chains ++ */ ++ dev_uc_flush(dev); ++ dev_mc_flush(dev); ++ ++ if (dev->netdev_ops->ndo_uninit) ++ dev->netdev_ops->ndo_uninit(dev); ++ ++ if (skb) ++ rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); ++ ++ /* Notifier chain MUST detach us all upper devices. */ ++ WARN_ON(netdev_has_any_upper_dev(dev)); ++ ++ /* Remove entries from kobject tree */ ++ netdev_unregister_kobject(dev); ++#ifdef CONFIG_XPS ++ /* Remove XPS queueing entries */ ++ netif_reset_xps_queues_gt(dev, 0); ++#endif ++ } ++ ++ synchronize_net(); ++ ++ list_for_each_entry(dev, head, unreg_list) ++ dev_put(dev); ++} ++ ++static void rollback_registered(struct net_device *dev) ++{ ++ LIST_HEAD(single); ++ ++ list_add(&dev->unreg_list, &single); ++ rollback_registered_many(&single); ++ list_del(&single); ++} ++ ++static netdev_features_t netdev_fix_features(struct net_device *dev, ++ netdev_features_t features) ++{ ++ /* Fix illegal checksum combinations */ ++ if ((features & NETIF_F_HW_CSUM) && ++ (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { ++ netdev_warn(dev, "mixed HW and IP checksum settings.\n"); ++ features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); ++ } ++ ++ /* TSO requires that SG is present as well. */ ++ if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { ++ netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); ++ features &= ~NETIF_F_ALL_TSO; ++ } ++ ++ if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && ++ !(features & NETIF_F_IP_CSUM)) { ++ netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); ++ features &= ~NETIF_F_TSO; ++ features &= ~NETIF_F_TSO_ECN; ++ } ++ ++ if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && ++ !(features & NETIF_F_IPV6_CSUM)) { ++ netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); ++ features &= ~NETIF_F_TSO6; ++ } ++ ++ /* TSO ECN requires that TSO is present as well. */ ++ if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) ++ features &= ~NETIF_F_TSO_ECN; ++ ++ /* Software GSO depends on SG. */ ++ if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { ++ netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); ++ features &= ~NETIF_F_GSO; ++ } ++ ++ /* UFO needs SG and checksumming */ ++ if (features & NETIF_F_UFO) { ++ /* maybe split UFO into V4 and V6? */ ++ if (!((features & NETIF_F_GEN_CSUM) || ++ (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM)) ++ == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { ++ netdev_dbg(dev, ++ "Dropping NETIF_F_UFO since no checksum offload features.\n"); ++ features &= ~NETIF_F_UFO; ++ } ++ ++ if (!(features & NETIF_F_SG)) { ++ netdev_dbg(dev, ++ "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n"); ++ features &= ~NETIF_F_UFO; ++ } ++ } ++ ++#ifdef CONFIG_NET_RX_BUSY_POLL ++ if (dev->netdev_ops->ndo_busy_poll) ++ features |= NETIF_F_BUSY_POLL; ++ else ++#endif ++ features &= ~NETIF_F_BUSY_POLL; ++ ++ return features; ++} ++ ++int __netdev_update_features(struct net_device *dev) ++{ ++ netdev_features_t features; ++ int err = 0; ++ ++ ASSERT_RTNL(); ++ ++ features = netdev_get_wanted_features(dev); ++ ++ if (dev->netdev_ops->ndo_fix_features) ++ features = dev->netdev_ops->ndo_fix_features(dev, features); ++ ++ /* driver might be less strict about feature dependencies */ ++ features = netdev_fix_features(dev, features); ++ ++ if (dev->features == features) ++ return 0; ++ ++ netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", ++ &dev->features, &features); ++ ++ if (dev->netdev_ops->ndo_set_features) ++ err = dev->netdev_ops->ndo_set_features(dev, features); ++ ++ if (unlikely(err < 0)) { ++ netdev_err(dev, ++ "set_features() failed (%d); wanted %pNF, left %pNF\n", ++ err, &features, &dev->features); ++ return -1; ++ } ++ ++ if (!err) ++ dev->features = features; ++ ++ return 1; ++} ++ ++/** ++ * netdev_update_features - recalculate device features ++ * @dev: the device to check ++ * ++ * Recalculate dev->features set and send notifications if it ++ * has changed. Should be called after driver or hardware dependent ++ * conditions might have changed that influence the features. ++ */ ++void netdev_update_features(struct net_device *dev) ++{ ++ if (__netdev_update_features(dev)) ++ netdev_features_change(dev); ++} ++EXPORT_SYMBOL(netdev_update_features); ++ ++/** ++ * netdev_change_features - recalculate device features ++ * @dev: the device to check ++ * ++ * Recalculate dev->features set and send notifications even ++ * if they have not changed. Should be called instead of ++ * netdev_update_features() if also dev->vlan_features might ++ * have changed to allow the changes to be propagated to stacked ++ * VLAN devices. ++ */ ++void netdev_change_features(struct net_device *dev) ++{ ++ __netdev_update_features(dev); ++ netdev_features_change(dev); ++} ++EXPORT_SYMBOL(netdev_change_features); ++ ++/** ++ * netif_stacked_transfer_operstate - transfer operstate ++ * @rootdev: the root or lower level device to transfer state from ++ * @dev: the device to transfer operstate to ++ * ++ * Transfer operational state from root to device. This is normally ++ * called when a stacking relationship exists between the root ++ * device and the device(a leaf device). ++ */ ++void netif_stacked_transfer_operstate(const struct net_device *rootdev, ++ struct net_device *dev) ++{ ++ if (rootdev->operstate == IF_OPER_DORMANT) ++ netif_dormant_on(dev); ++ else ++ netif_dormant_off(dev); ++ ++ if (netif_carrier_ok(rootdev)) { ++ if (!netif_carrier_ok(dev)) ++ netif_carrier_on(dev); ++ } else { ++ if (netif_carrier_ok(dev)) ++ netif_carrier_off(dev); ++ } ++} ++EXPORT_SYMBOL(netif_stacked_transfer_operstate); ++ ++#ifdef CONFIG_SYSFS ++static int netif_alloc_rx_queues(struct net_device *dev) ++{ ++ unsigned int i, count = dev->num_rx_queues; ++ struct netdev_rx_queue *rx; ++ size_t sz = count * sizeof(*rx); ++ ++ BUG_ON(count < 1); ++ ++ rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); ++ if (!rx) { ++ rx = vzalloc(sz); ++ if (!rx) ++ return -ENOMEM; ++ } ++ dev->_rx = rx; ++ ++ for (i = 0; i < count; i++) ++ rx[i].dev = dev; ++ return 0; ++} ++#endif ++ ++static void netdev_init_one_queue(struct net_device *dev, ++ struct netdev_queue *queue, void *_unused) ++{ ++ /* Initialize queue lock */ ++ spin_lock_init(&queue->_xmit_lock); ++ netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); ++ queue->xmit_lock_owner = -1; ++ netdev_queue_numa_node_write(queue, NUMA_NO_NODE); ++ queue->dev = dev; ++#ifdef CONFIG_BQL ++ dql_init(&queue->dql, HZ); ++#endif ++} ++ ++static void netif_free_tx_queues(struct net_device *dev) ++{ ++ kvfree(dev->_tx); ++} ++ ++static int netif_alloc_netdev_queues(struct net_device *dev) ++{ ++ unsigned int count = dev->num_tx_queues; ++ struct netdev_queue *tx; ++ size_t sz = count * sizeof(*tx); ++ ++ if (count < 1 || count > 0xffff) ++ return -EINVAL; ++ ++ tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); ++ if (!tx) { ++ tx = vzalloc(sz); ++ if (!tx) ++ return -ENOMEM; ++ } ++ dev->_tx = tx; ++ ++ netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); ++ spin_lock_init(&dev->tx_global_lock); ++ ++ return 0; ++} ++ ++/** ++ * register_netdevice - register a network device ++ * @dev: device to register ++ * ++ * Take a completed network device structure and add it to the kernel ++ * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier ++ * chain. 0 is returned on success. A negative errno code is returned ++ * on a failure to set up the device, or if the name is a duplicate. ++ * ++ * Callers must hold the rtnl semaphore. You may want ++ * register_netdev() instead of this. ++ * ++ * BUGS: ++ * The locking appears insufficient to guarantee two parallel registers ++ * will not get the same name. ++ */ ++ ++int register_netdevice(struct net_device *dev) ++{ ++ int ret; ++ struct net *net = dev_net(dev); ++ ++ BUG_ON(dev_boot_phase); ++ ASSERT_RTNL(); ++ ++ might_sleep(); ++ ++ /* When net_device's are persistent, this will be fatal. */ ++ BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); ++ BUG_ON(!net); ++ ++ spin_lock_init(&dev->addr_list_lock); ++ netdev_set_addr_lockdep_class(dev); ++ ++ ret = dev_get_valid_name(net, dev, dev->name); ++ if (ret < 0) ++ goto out; ++ ++ /* Init, if this function is available */ ++ if (dev->netdev_ops->ndo_init) { ++ ret = dev->netdev_ops->ndo_init(dev); ++ if (ret) { ++ if (ret > 0) ++ ret = -EIO; ++ goto out; ++ } ++ } ++ ++ if (((dev->hw_features | dev->features) & ++ NETIF_F_HW_VLAN_CTAG_FILTER) && ++ (!dev->netdev_ops->ndo_vlan_rx_add_vid || ++ !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { ++ netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); ++ ret = -EINVAL; ++ goto err_uninit; ++ } ++ ++ ret = -EBUSY; ++ if (!dev->ifindex) ++ dev->ifindex = dev_new_index(net); ++ else if (__dev_get_by_index(net, dev->ifindex)) ++ goto err_uninit; ++ ++ /* Transfer changeable features to wanted_features and enable ++ * software offloads (GSO and GRO). ++ */ ++ dev->hw_features |= NETIF_F_SOFT_FEATURES; ++ dev->features |= NETIF_F_SOFT_FEATURES; ++ dev->wanted_features = dev->features & dev->hw_features; ++ ++ if (!(dev->flags & IFF_LOOPBACK)) { ++ dev->hw_features |= NETIF_F_NOCACHE_COPY; ++ } ++ ++ /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. ++ */ ++ dev->vlan_features |= NETIF_F_HIGHDMA; ++ ++ /* Make NETIF_F_SG inheritable to tunnel devices. ++ */ ++ dev->hw_enc_features |= NETIF_F_SG; ++ ++ /* Make NETIF_F_SG inheritable to MPLS. ++ */ ++ dev->mpls_features |= NETIF_F_SG; ++ ++ ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); ++ ret = notifier_to_errno(ret); ++ if (ret) ++ goto err_uninit; ++ ++ ret = netdev_register_kobject(dev); ++ if (ret) ++ goto err_uninit; ++ dev->reg_state = NETREG_REGISTERED; ++ ++ __netdev_update_features(dev); ++ ++ /* ++ * Default initial state at registry is that the ++ * device is present. ++ */ ++ ++ set_bit(__LINK_STATE_PRESENT, &dev->state); ++ ++ linkwatch_init_dev(dev); ++ ++ dev_init_scheduler(dev); ++ dev_hold(dev); ++ list_netdevice(dev); ++ add_device_randomness(dev->dev_addr, dev->addr_len); ++ ++ /* If the device has permanent device address, driver should ++ * set dev_addr and also addr_assign_type should be set to ++ * NET_ADDR_PERM (default value). ++ */ ++ if (dev->addr_assign_type == NET_ADDR_PERM) ++ memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); ++ ++ /* Notify protocols, that a new device appeared. */ ++ ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); ++ ret = notifier_to_errno(ret); ++ if (ret) { ++ rollback_registered(dev); ++ dev->reg_state = NETREG_UNREGISTERED; ++ } ++ /* ++ * Prevent userspace races by waiting until the network ++ * device is fully setup before sending notifications. ++ */ ++ if (!dev->rtnl_link_ops || ++ dev->rtnl_link_state == RTNL_LINK_INITIALIZED) ++ rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); ++ ++out: ++ return ret; ++ ++err_uninit: ++ if (dev->netdev_ops->ndo_uninit) ++ dev->netdev_ops->ndo_uninit(dev); ++ goto out; ++} ++EXPORT_SYMBOL(register_netdevice); ++ ++/** ++ * init_dummy_netdev - init a dummy network device for NAPI ++ * @dev: device to init ++ * ++ * This takes a network device structure and initialize the minimum ++ * amount of fields so it can be used to schedule NAPI polls without ++ * registering a full blown interface. This is to be used by drivers ++ * that need to tie several hardware interfaces to a single NAPI ++ * poll scheduler due to HW limitations. ++ */ ++int init_dummy_netdev(struct net_device *dev) ++{ ++ /* Clear everything. Note we don't initialize spinlocks ++ * are they aren't supposed to be taken by any of the ++ * NAPI code and this dummy netdev is supposed to be ++ * only ever used for NAPI polls ++ */ ++ memset(dev, 0, sizeof(struct net_device)); ++ ++ /* make sure we BUG if trying to hit standard ++ * register/unregister code path ++ */ ++ dev->reg_state = NETREG_DUMMY; ++ ++ /* NAPI wants this */ ++ INIT_LIST_HEAD(&dev->napi_list); ++ ++ /* a dummy interface is started by default */ ++ set_bit(__LINK_STATE_PRESENT, &dev->state); ++ set_bit(__LINK_STATE_START, &dev->state); ++ ++ /* Note : We dont allocate pcpu_refcnt for dummy devices, ++ * because users of this 'device' dont need to change ++ * its refcount. ++ */ ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(init_dummy_netdev); ++ ++ ++/** ++ * register_netdev - register a network device ++ * @dev: device to register ++ * ++ * Take a completed network device structure and add it to the kernel ++ * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier ++ * chain. 0 is returned on success. A negative errno code is returned ++ * on a failure to set up the device, or if the name is a duplicate. ++ * ++ * This is a wrapper around register_netdevice that takes the rtnl semaphore ++ * and expands the device name if you passed a format string to ++ * alloc_netdev. ++ */ ++int register_netdev(struct net_device *dev) ++{ ++ int err; ++ ++ rtnl_lock(); ++ err = register_netdevice(dev); ++ rtnl_unlock(); ++ return err; ++} ++EXPORT_SYMBOL(register_netdev); ++ ++int netdev_refcnt_read(const struct net_device *dev) ++{ ++ int i, refcnt = 0; ++ ++ for_each_possible_cpu(i) ++ refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); ++ return refcnt; ++} ++EXPORT_SYMBOL(netdev_refcnt_read); ++ ++/** ++ * netdev_wait_allrefs - wait until all references are gone. ++ * @dev: target net_device ++ * ++ * This is called when unregistering network devices. ++ * ++ * Any protocol or device that holds a reference should register ++ * for netdevice notification, and cleanup and put back the ++ * reference if they receive an UNREGISTER event. ++ * We can get stuck here if buggy protocols don't correctly ++ * call dev_put. ++ */ ++static void netdev_wait_allrefs(struct net_device *dev) ++{ ++ unsigned long rebroadcast_time, warning_time; ++ int refcnt; ++ ++ linkwatch_forget_dev(dev); ++ ++ rebroadcast_time = warning_time = jiffies; ++ refcnt = netdev_refcnt_read(dev); ++ ++ while (refcnt != 0) { ++ if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { ++ rtnl_lock(); ++ ++ /* Rebroadcast unregister notification */ ++ call_netdevice_notifiers(NETDEV_UNREGISTER, dev); ++ ++ __rtnl_unlock(); ++ rcu_barrier(); ++ rtnl_lock(); ++ ++ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); ++ if (test_bit(__LINK_STATE_LINKWATCH_PENDING, ++ &dev->state)) { ++ /* We must not have linkwatch events ++ * pending on unregister. If this ++ * happens, we simply run the queue ++ * unscheduled, resulting in a noop ++ * for this device. ++ */ ++ linkwatch_run_queue(); ++ } ++ ++ __rtnl_unlock(); ++ ++ rebroadcast_time = jiffies; ++ } ++ ++ msleep(250); ++ ++ refcnt = netdev_refcnt_read(dev); ++ ++ if (time_after(jiffies, warning_time + 10 * HZ)) { ++ pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", ++ dev->name, refcnt); ++ warning_time = jiffies; ++ } ++ } ++} ++ ++/* The sequence is: ++ * ++ * rtnl_lock(); ++ * ... ++ * register_netdevice(x1); ++ * register_netdevice(x2); ++ * ... ++ * unregister_netdevice(y1); ++ * unregister_netdevice(y2); ++ * ... ++ * rtnl_unlock(); ++ * free_netdev(y1); ++ * free_netdev(y2); ++ * ++ * We are invoked by rtnl_unlock(). ++ * This allows us to deal with problems: ++ * 1) We can delete sysfs objects which invoke hotplug ++ * without deadlocking with linkwatch via keventd. ++ * 2) Since we run with the RTNL semaphore not held, we can sleep ++ * safely in order to wait for the netdev refcnt to drop to zero. ++ * ++ * We must not return until all unregister events added during ++ * the interval the lock was held have been completed. ++ */ ++void netdev_run_todo(void) ++{ ++ struct list_head list; ++ ++ /* Snapshot list, allow later requests */ ++ list_replace_init(&net_todo_list, &list); ++ ++ __rtnl_unlock(); ++ ++ ++ /* Wait for rcu callbacks to finish before next phase */ ++ if (!list_empty(&list)) ++ rcu_barrier(); ++ ++ while (!list_empty(&list)) { ++ struct net_device *dev ++ = list_first_entry(&list, struct net_device, todo_list); ++ list_del(&dev->todo_list); ++ ++ rtnl_lock(); ++ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); ++ __rtnl_unlock(); ++ ++ if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { ++ pr_err("network todo '%s' but state %d\n", ++ dev->name, dev->reg_state); ++ dump_stack(); ++ continue; ++ } ++ ++ dev->reg_state = NETREG_UNREGISTERED; ++ ++ netdev_wait_allrefs(dev); ++ ++ /* paranoia */ ++ BUG_ON(netdev_refcnt_read(dev)); ++ BUG_ON(!list_empty(&dev->ptype_all)); ++ BUG_ON(!list_empty(&dev->ptype_specific)); ++ WARN_ON(rcu_access_pointer(dev->ip_ptr)); ++ WARN_ON(rcu_access_pointer(dev->ip6_ptr)); ++ WARN_ON(dev->dn_ptr); ++ ++ if (dev->destructor) ++ dev->destructor(dev); ++ ++ /* Report a network device has been unregistered */ ++ rtnl_lock(); ++ dev_net(dev)->dev_unreg_count--; ++ __rtnl_unlock(); ++ wake_up(&netdev_unregistering_wq); ++ ++ /* Free network device */ ++ kobject_put(&dev->dev.kobj); ++ } ++} ++ ++/* Convert net_device_stats to rtnl_link_stats64. They have the same ++ * fields in the same order, with only the type differing. ++ */ ++void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, ++ const struct net_device_stats *netdev_stats) ++{ ++#if BITS_PER_LONG == 64 ++ BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats)); ++ memcpy(stats64, netdev_stats, sizeof(*stats64)); ++#else ++ size_t i, n = sizeof(*stats64) / sizeof(u64); ++ const unsigned long *src = (const unsigned long *)netdev_stats; ++ u64 *dst = (u64 *)stats64; ++ ++ BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) != ++ sizeof(*stats64) / sizeof(u64)); ++ for (i = 0; i < n; i++) ++ dst[i] = src[i]; ++#endif ++} ++EXPORT_SYMBOL(netdev_stats_to_stats64); ++ ++/** ++ * dev_get_stats - get network device statistics ++ * @dev: device to get statistics from ++ * @storage: place to store stats ++ * ++ * Get network statistics from device. Return @storage. ++ * The device driver may provide its own method by setting ++ * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; ++ * otherwise the internal statistics structure is used. ++ */ ++struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, ++ struct rtnl_link_stats64 *storage) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (ops->ndo_get_stats64) { ++ memset(storage, 0, sizeof(*storage)); ++ ops->ndo_get_stats64(dev, storage); ++ } else if (ops->ndo_get_stats) { ++ netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); ++ } else { ++ netdev_stats_to_stats64(storage, &dev->stats); ++ } ++ storage->rx_dropped += atomic_long_read(&dev->rx_dropped); ++ storage->tx_dropped += atomic_long_read(&dev->tx_dropped); ++ return storage; ++} ++EXPORT_SYMBOL(dev_get_stats); ++ ++struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) ++{ ++ struct netdev_queue *queue = dev_ingress_queue(dev); ++ ++#ifdef CONFIG_NET_CLS_ACT ++ if (queue) ++ return queue; ++ queue = kzalloc(sizeof(*queue), GFP_KERNEL); ++ if (!queue) ++ return NULL; ++ netdev_init_one_queue(dev, queue, NULL); ++ RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); ++ queue->qdisc_sleeping = &noop_qdisc; ++ rcu_assign_pointer(dev->ingress_queue, queue); ++#endif ++ return queue; ++} ++ ++static const struct ethtool_ops default_ethtool_ops; ++ ++void netdev_set_default_ethtool_ops(struct net_device *dev, ++ const struct ethtool_ops *ops) ++{ ++ if (dev->ethtool_ops == &default_ethtool_ops) ++ dev->ethtool_ops = ops; ++} ++EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); ++ ++void netdev_freemem(struct net_device *dev) ++{ ++ char *addr = (char *)dev - dev->padded; ++ ++ kvfree(addr); ++} ++ ++/** ++ * alloc_netdev_mqs - allocate network device ++ * @sizeof_priv: size of private data to allocate space for ++ * @name: device name format string ++ * @name_assign_type: origin of device name ++ * @setup: callback to initialize device ++ * @txqs: the number of TX subqueues to allocate ++ * @rxqs: the number of RX subqueues to allocate ++ * ++ * Allocates a struct net_device with private data area for driver use ++ * and performs basic initialization. Also allocates subqueue structs ++ * for each queue on the device. ++ */ ++struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, ++ unsigned char name_assign_type, ++ void (*setup)(struct net_device *), ++ unsigned int txqs, unsigned int rxqs) ++{ ++ struct net_device *dev; ++ size_t alloc_size; ++ struct net_device *p; ++ ++ BUG_ON(strlen(name) >= sizeof(dev->name)); ++ ++ if (txqs < 1) { ++ pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); ++ return NULL; ++ } ++ ++#ifdef CONFIG_SYSFS ++ if (rxqs < 1) { ++ pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); ++ return NULL; ++ } ++#endif ++ ++ alloc_size = sizeof(struct net_device); ++ if (sizeof_priv) { ++ /* ensure 32-byte alignment of private area */ ++ alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); ++ alloc_size += sizeof_priv; ++ } ++ /* ensure 32-byte alignment of whole construct */ ++ alloc_size += NETDEV_ALIGN - 1; ++ ++ p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); ++ if (!p) ++ p = vzalloc(alloc_size); ++ if (!p) ++ return NULL; ++ ++ dev = PTR_ALIGN(p, NETDEV_ALIGN); ++ dev->padded = (char *)dev - (char *)p; ++ ++ dev->pcpu_refcnt = alloc_percpu(int); ++ if (!dev->pcpu_refcnt) ++ goto free_dev; ++ ++ if (dev_addr_init(dev)) ++ goto free_pcpu; ++ ++ dev_mc_init(dev); ++ dev_uc_init(dev); ++ ++ dev_net_set(dev, &init_net); ++ ++ dev->gso_max_size = GSO_MAX_SIZE; ++ dev->gso_max_segs = GSO_MAX_SEGS; ++ dev->gso_min_segs = 0; ++ ++ INIT_LIST_HEAD(&dev->napi_list); ++ INIT_LIST_HEAD(&dev->unreg_list); ++ INIT_LIST_HEAD(&dev->close_list); ++ INIT_LIST_HEAD(&dev->link_watch_list); ++ INIT_LIST_HEAD(&dev->adj_list.upper); ++ INIT_LIST_HEAD(&dev->adj_list.lower); ++ INIT_LIST_HEAD(&dev->all_adj_list.upper); ++ INIT_LIST_HEAD(&dev->all_adj_list.lower); ++ INIT_LIST_HEAD(&dev->ptype_all); ++ INIT_LIST_HEAD(&dev->ptype_specific); ++ dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; ++ setup(dev); ++ ++ dev->num_tx_queues = txqs; ++ dev->real_num_tx_queues = txqs; ++ if (netif_alloc_netdev_queues(dev)) ++ goto free_all; ++ ++#ifdef CONFIG_SYSFS ++ dev->num_rx_queues = rxqs; ++ dev->real_num_rx_queues = rxqs; ++ if (netif_alloc_rx_queues(dev)) ++ goto free_all; ++#endif ++ ++ strcpy(dev->name, name); ++ dev->name_assign_type = name_assign_type; ++ dev->group = INIT_NETDEV_GROUP; ++ if (!dev->ethtool_ops) ++ dev->ethtool_ops = &default_ethtool_ops; ++ return dev; ++ ++free_all: ++ free_netdev(dev); ++ return NULL; ++ ++free_pcpu: ++ free_percpu(dev->pcpu_refcnt); ++free_dev: ++ netdev_freemem(dev); ++ return NULL; ++} ++EXPORT_SYMBOL(alloc_netdev_mqs); ++ ++/** ++ * free_netdev - free network device ++ * @dev: device ++ * ++ * This function does the last stage of destroying an allocated device ++ * interface. The reference to the device object is released. ++ * If this is the last reference then it will be freed. ++ */ ++void free_netdev(struct net_device *dev) ++{ ++ struct napi_struct *p, *n; ++ ++ netif_free_tx_queues(dev); ++#ifdef CONFIG_SYSFS ++ kvfree(dev->_rx); ++#endif ++ ++ kfree(rcu_dereference_protected(dev->ingress_queue, 1)); ++ ++ /* Flush device addresses */ ++ dev_addr_flush(dev); ++ ++ list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) ++ netif_napi_del(p); ++ ++ free_percpu(dev->pcpu_refcnt); ++ dev->pcpu_refcnt = NULL; ++ ++ /* Compatibility with error handling in drivers */ ++ if (dev->reg_state == NETREG_UNINITIALIZED) { ++ netdev_freemem(dev); ++ return; ++ } ++ ++ BUG_ON(dev->reg_state != NETREG_UNREGISTERED); ++ dev->reg_state = NETREG_RELEASED; ++ ++ /* will free via device release */ ++ put_device(&dev->dev); ++} ++EXPORT_SYMBOL(free_netdev); ++ ++/** ++ * synchronize_net - Synchronize with packet receive processing ++ * ++ * Wait for packets currently being received to be done. ++ * Does not block later packets from starting. ++ */ ++void synchronize_net(void) ++{ ++ might_sleep(); ++ if (rtnl_is_locked()) ++ synchronize_rcu_expedited(); ++ else ++ synchronize_rcu(); ++} ++EXPORT_SYMBOL(synchronize_net); ++ ++/** ++ * unregister_netdevice_queue - remove device from the kernel ++ * @dev: device ++ * @head: list ++ * ++ * This function shuts down a device interface and removes it ++ * from the kernel tables. ++ * If head not NULL, device is queued to be unregistered later. ++ * ++ * Callers must hold the rtnl semaphore. You may want ++ * unregister_netdev() instead of this. ++ */ ++ ++void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) ++{ ++ ASSERT_RTNL(); ++ ++ if (head) { ++ list_move_tail(&dev->unreg_list, head); ++ } else { ++ rollback_registered(dev); ++ /* Finish processing unregister after unlock */ ++ net_set_todo(dev); ++ } ++} ++EXPORT_SYMBOL(unregister_netdevice_queue); ++ ++/** ++ * unregister_netdevice_many - unregister many devices ++ * @head: list of devices ++ * ++ * Note: As most callers use a stack allocated list_head, ++ * we force a list_del() to make sure stack wont be corrupted later. ++ */ ++void unregister_netdevice_many(struct list_head *head) ++{ ++ struct net_device *dev; ++ ++ if (!list_empty(head)) { ++ rollback_registered_many(head); ++ list_for_each_entry(dev, head, unreg_list) ++ net_set_todo(dev); ++ list_del(head); ++ } ++} ++EXPORT_SYMBOL(unregister_netdevice_many); ++ ++/** ++ * unregister_netdev - remove device from the kernel ++ * @dev: device ++ * ++ * This function shuts down a device interface and removes it ++ * from the kernel tables. ++ * ++ * This is just a wrapper for unregister_netdevice that takes ++ * the rtnl semaphore. In general you want to use this and not ++ * unregister_netdevice. ++ */ ++void unregister_netdev(struct net_device *dev) ++{ ++ rtnl_lock(); ++ unregister_netdevice(dev); ++ rtnl_unlock(); ++} ++EXPORT_SYMBOL(unregister_netdev); ++ ++/** ++ * dev_change_net_namespace - move device to different nethost namespace ++ * @dev: device ++ * @net: network namespace ++ * @pat: If not NULL name pattern to try if the current device name ++ * is already taken in the destination network namespace. ++ * ++ * This function shuts down a device interface and moves it ++ * to a new network namespace. On success 0 is returned, on ++ * a failure a netagive errno code is returned. ++ * ++ * Callers must hold the rtnl semaphore. ++ */ ++ ++int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) ++{ ++ int err; ++ ++ ASSERT_RTNL(); ++ ++ /* Don't allow namespace local devices to be moved. */ ++ err = -EINVAL; ++ if (dev->features & NETIF_F_NETNS_LOCAL) ++ goto out; ++ ++ /* Ensure the device has been registrered */ ++ if (dev->reg_state != NETREG_REGISTERED) ++ goto out; ++ ++ /* Get out if there is nothing todo */ ++ err = 0; ++ if (net_eq(dev_net(dev), net)) ++ goto out; ++ ++ /* Pick the destination device name, and ensure ++ * we can use it in the destination network namespace. ++ */ ++ err = -EEXIST; ++ if (__dev_get_by_name(net, dev->name)) { ++ /* We get here if we can't use the current device name */ ++ if (!pat) ++ goto out; ++ if (dev_get_valid_name(net, dev, pat) < 0) ++ goto out; ++ } ++ ++ /* ++ * And now a mini version of register_netdevice unregister_netdevice. ++ */ ++ ++ /* If device is running close it first. */ ++ dev_close(dev); ++ ++ /* And unlink it from device chain */ ++ err = -ENODEV; ++ unlist_netdevice(dev); ++ ++ synchronize_net(); ++ ++ /* Shutdown queueing discipline. */ ++ dev_shutdown(dev); ++ ++ /* Notify protocols, that we are about to destroy ++ this device. They should clean all the things. ++ ++ Note that dev->reg_state stays at NETREG_REGISTERED. ++ This is wanted because this way 8021q and macvlan know ++ the device is just moving and can keep their slaves up. ++ */ ++ call_netdevice_notifiers(NETDEV_UNREGISTER, dev); ++ rcu_barrier(); ++ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); ++ rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL); ++ ++ /* ++ * Flush the unicast and multicast chains ++ */ ++ dev_uc_flush(dev); ++ dev_mc_flush(dev); ++ ++ /* Send a netdev-removed uevent to the old namespace */ ++ kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); ++ netdev_adjacent_del_links(dev); ++ ++ /* Actually switch the network namespace */ ++ dev_net_set(dev, net); ++ ++ /* If there is an ifindex conflict assign a new one */ ++ if (__dev_get_by_index(net, dev->ifindex)) ++ dev->ifindex = dev_new_index(net); ++ ++ /* Send a netdev-add uevent to the new namespace */ ++ kobject_uevent(&dev->dev.kobj, KOBJ_ADD); ++ netdev_adjacent_add_links(dev); ++ ++ /* Fixup kobjects */ ++ err = device_rename(&dev->dev, dev->name); ++ WARN_ON(err); ++ ++ /* Add the device back in the hashes */ ++ list_netdevice(dev); ++ ++ /* Notify protocols, that a new device appeared. */ ++ call_netdevice_notifiers(NETDEV_REGISTER, dev); ++ ++ /* ++ * Prevent userspace races by waiting until the network ++ * device is fully setup before sending notifications. ++ */ ++ rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); ++ ++ synchronize_net(); ++ err = 0; ++out: ++ return err; ++} ++EXPORT_SYMBOL_GPL(dev_change_net_namespace); ++ ++static int dev_cpu_callback(struct notifier_block *nfb, ++ unsigned long action, ++ void *ocpu) ++{ ++ struct sk_buff **list_skb; ++ struct sk_buff *skb; ++ unsigned int cpu, oldcpu = (unsigned long)ocpu; ++ struct softnet_data *sd, *oldsd; ++ ++ if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) ++ return NOTIFY_OK; ++ ++ local_irq_disable(); ++ cpu = smp_processor_id(); ++ sd = &per_cpu(softnet_data, cpu); ++ oldsd = &per_cpu(softnet_data, oldcpu); ++ ++ /* Find end of our completion_queue. */ ++ list_skb = &sd->completion_queue; ++ while (*list_skb) ++ list_skb = &(*list_skb)->next; ++ /* Append completion queue from offline CPU. */ ++ *list_skb = oldsd->completion_queue; ++ oldsd->completion_queue = NULL; ++ ++ /* Append output queue from offline CPU. */ ++ if (oldsd->output_queue) { ++ *sd->output_queue_tailp = oldsd->output_queue; ++ sd->output_queue_tailp = oldsd->output_queue_tailp; ++ oldsd->output_queue = NULL; ++ oldsd->output_queue_tailp = &oldsd->output_queue; ++ } ++ /* Append NAPI poll list from offline CPU, with one exception : ++ * process_backlog() must be called by cpu owning percpu backlog. ++ * We properly handle process_queue & input_pkt_queue later. ++ */ ++ while (!list_empty(&oldsd->poll_list)) { ++ struct napi_struct *napi = list_first_entry(&oldsd->poll_list, ++ struct napi_struct, ++ poll_list); ++ ++ list_del_init(&napi->poll_list); ++ if (napi->poll == process_backlog) ++ napi->state = 0; ++ else ++ ____napi_schedule(sd, napi); ++ } ++ ++ raise_softirq_irqoff(NET_TX_SOFTIRQ); ++ local_irq_enable(); ++ ++ /* Process offline CPU's input_pkt_queue */ ++ while ((skb = __skb_dequeue(&oldsd->process_queue))) { ++ netif_rx_ni(skb); ++ input_queue_head_incr(oldsd); ++ } ++ while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { ++ netif_rx_ni(skb); ++ input_queue_head_incr(oldsd); ++ } ++ ++ return NOTIFY_OK; ++} ++ ++ ++/** ++ * netdev_increment_features - increment feature set by one ++ * @all: current feature set ++ * @one: new feature set ++ * @mask: mask feature set ++ * ++ * Computes a new feature set after adding a device with feature set ++ * @one to the master device with current feature set @all. Will not ++ * enable anything that is off in @mask. Returns the new feature set. ++ */ ++netdev_features_t netdev_increment_features(netdev_features_t all, ++ netdev_features_t one, netdev_features_t mask) ++{ ++ if (mask & NETIF_F_GEN_CSUM) ++ mask |= NETIF_F_ALL_CSUM; ++ mask |= NETIF_F_VLAN_CHALLENGED; ++ ++ all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask; ++ all &= one | ~NETIF_F_ALL_FOR_ALL; ++ ++ /* If one device supports hw checksumming, set for all. */ ++ if (all & NETIF_F_GEN_CSUM) ++ all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM); ++ ++ return all; ++} ++EXPORT_SYMBOL(netdev_increment_features); ++ ++static struct hlist_head * __net_init netdev_create_hash(void) ++{ ++ int i; ++ struct hlist_head *hash; ++ ++ hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); ++ if (hash != NULL) ++ for (i = 0; i < NETDEV_HASHENTRIES; i++) ++ INIT_HLIST_HEAD(&hash[i]); ++ ++ return hash; ++} ++ ++/* Initialize per network namespace state */ ++static int __net_init netdev_init(struct net *net) ++{ ++ if (net != &init_net) ++ INIT_LIST_HEAD(&net->dev_base_head); ++ ++ net->dev_name_head = netdev_create_hash(); ++ if (net->dev_name_head == NULL) ++ goto err_name; ++ ++ net->dev_index_head = netdev_create_hash(); ++ if (net->dev_index_head == NULL) ++ goto err_idx; ++ ++ return 0; ++ ++err_idx: ++ kfree(net->dev_name_head); ++err_name: ++ return -ENOMEM; ++} ++ ++/** ++ * netdev_drivername - network driver for the device ++ * @dev: network device ++ * ++ * Determine network driver for device. ++ */ ++const char *netdev_drivername(const struct net_device *dev) ++{ ++ const struct device_driver *driver; ++ const struct device *parent; ++ const char *empty = ""; ++ ++ parent = dev->dev.parent; ++ if (!parent) ++ return empty; ++ ++ driver = parent->driver; ++ if (driver && driver->name) ++ return driver->name; ++ return empty; ++} ++ ++static void __netdev_printk(const char *level, const struct net_device *dev, ++ struct va_format *vaf) ++{ ++ if (dev && dev->dev.parent) { ++ dev_printk_emit(level[1] - '0', ++ dev->dev.parent, ++ "%s %s %s%s: %pV", ++ dev_driver_string(dev->dev.parent), ++ dev_name(dev->dev.parent), ++ netdev_name(dev), netdev_reg_state(dev), ++ vaf); ++ } else if (dev) { ++ printk("%s%s%s: %pV", ++ level, netdev_name(dev), netdev_reg_state(dev), vaf); ++ } else { ++ printk("%s(NULL net_device): %pV", level, vaf); ++ } ++} ++ ++void netdev_printk(const char *level, const struct net_device *dev, ++ const char *format, ...) ++{ ++ struct va_format vaf; ++ va_list args; ++ ++ va_start(args, format); ++ ++ vaf.fmt = format; ++ vaf.va = &args; ++ ++ __netdev_printk(level, dev, &vaf); ++ ++ va_end(args); ++} ++EXPORT_SYMBOL(netdev_printk); ++ ++#define define_netdev_printk_level(func, level) \ ++void func(const struct net_device *dev, const char *fmt, ...) \ ++{ \ ++ struct va_format vaf; \ ++ va_list args; \ ++ \ ++ va_start(args, fmt); \ ++ \ ++ vaf.fmt = fmt; \ ++ vaf.va = &args; \ ++ \ ++ __netdev_printk(level, dev, &vaf); \ ++ \ ++ va_end(args); \ ++} \ ++EXPORT_SYMBOL(func); ++ ++define_netdev_printk_level(netdev_emerg, KERN_EMERG); ++define_netdev_printk_level(netdev_alert, KERN_ALERT); ++define_netdev_printk_level(netdev_crit, KERN_CRIT); ++define_netdev_printk_level(netdev_err, KERN_ERR); ++define_netdev_printk_level(netdev_warn, KERN_WARNING); ++define_netdev_printk_level(netdev_notice, KERN_NOTICE); ++define_netdev_printk_level(netdev_info, KERN_INFO); ++ ++static void __net_exit netdev_exit(struct net *net) ++{ ++ kfree(net->dev_name_head); ++ kfree(net->dev_index_head); ++} ++ ++static struct pernet_operations __net_initdata netdev_net_ops = { ++ .init = netdev_init, ++ .exit = netdev_exit, ++}; ++ ++static void __net_exit default_device_exit(struct net *net) ++{ ++ struct net_device *dev, *aux; ++ /* ++ * Push all migratable network devices back to the ++ * initial network namespace ++ */ ++ rtnl_lock(); ++ for_each_netdev_safe(net, dev, aux) { ++ int err; ++ char fb_name[IFNAMSIZ]; ++ ++ /* Ignore unmoveable devices (i.e. loopback) */ ++ if (dev->features & NETIF_F_NETNS_LOCAL) ++ continue; ++ ++ /* Leave virtual devices for the generic cleanup */ ++ if (dev->rtnl_link_ops) ++ continue; ++ ++ /* Push remaining network devices to init_net */ ++ snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); ++ err = dev_change_net_namespace(dev, &init_net, fb_name); ++ if (err) { ++ pr_emerg("%s: failed to move %s to init_net: %d\n", ++ __func__, dev->name, err); ++ BUG(); ++ } ++ } ++ rtnl_unlock(); ++} ++ ++static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) ++{ ++ /* Return with the rtnl_lock held when there are no network ++ * devices unregistering in any network namespace in net_list. ++ */ ++ struct net *net; ++ bool unregistering; ++ DEFINE_WAIT_FUNC(wait, woken_wake_function); ++ ++ add_wait_queue(&netdev_unregistering_wq, &wait); ++ for (;;) { ++ unregistering = false; ++ rtnl_lock(); ++ list_for_each_entry(net, net_list, exit_list) { ++ if (net->dev_unreg_count > 0) { ++ unregistering = true; ++ break; ++ } ++ } ++ if (!unregistering) ++ break; ++ __rtnl_unlock(); ++ ++ wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); ++ } ++ remove_wait_queue(&netdev_unregistering_wq, &wait); ++} ++ ++static void __net_exit default_device_exit_batch(struct list_head *net_list) ++{ ++ /* At exit all network devices most be removed from a network ++ * namespace. Do this in the reverse order of registration. ++ * Do this across as many network namespaces as possible to ++ * improve batching efficiency. ++ */ ++ struct net_device *dev; ++ struct net *net; ++ LIST_HEAD(dev_kill_list); ++ ++ /* To prevent network device cleanup code from dereferencing ++ * loopback devices or network devices that have been freed ++ * wait here for all pending unregistrations to complete, ++ * before unregistring the loopback device and allowing the ++ * network namespace be freed. ++ * ++ * The netdev todo list containing all network devices ++ * unregistrations that happen in default_device_exit_batch ++ * will run in the rtnl_unlock() at the end of ++ * default_device_exit_batch. ++ */ ++ rtnl_lock_unregistering(net_list); ++ list_for_each_entry(net, net_list, exit_list) { ++ for_each_netdev_reverse(net, dev) { ++ if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) ++ dev->rtnl_link_ops->dellink(dev, &dev_kill_list); ++ else ++ unregister_netdevice_queue(dev, &dev_kill_list); ++ } ++ } ++ unregister_netdevice_many(&dev_kill_list); ++ rtnl_unlock(); ++} ++ ++static struct pernet_operations __net_initdata default_device_ops = { ++ .exit = default_device_exit, ++ .exit_batch = default_device_exit_batch, ++}; ++ ++/* ++ * Initialize the DEV module. At boot time this walks the device list and ++ * unhooks any devices that fail to initialise (normally hardware not ++ * present) and leaves us with a valid list of present and active devices. ++ * ++ */ ++ ++/* ++ * This is called single threaded during boot, so no need ++ * to take the rtnl semaphore. ++ */ ++static int __init net_dev_init(void) ++{ ++ int i, rc = -ENOMEM; ++ ++ BUG_ON(!dev_boot_phase); ++ ++ if (dev_proc_init()) ++ goto out; ++ ++ if (netdev_kobject_init()) ++ goto out; ++ ++ INIT_LIST_HEAD(&ptype_all); ++ for (i = 0; i < PTYPE_HASH_SIZE; i++) ++ INIT_LIST_HEAD(&ptype_base[i]); ++ ++ INIT_LIST_HEAD(&offload_base); ++ ++ if (register_pernet_subsys(&netdev_net_ops)) ++ goto out; ++ ++ /* ++ * Initialise the packet receive queues. ++ */ ++ ++ for_each_possible_cpu(i) { ++ struct softnet_data *sd = &per_cpu(softnet_data, i); ++ ++ skb_queue_head_init(&sd->input_pkt_queue); ++ skb_queue_head_init(&sd->process_queue); ++ INIT_LIST_HEAD(&sd->poll_list); ++ sd->output_queue_tailp = &sd->output_queue; ++#ifdef CONFIG_RPS ++ sd->csd.func = rps_trigger_softirq; ++ sd->csd.info = sd; ++ sd->cpu = i; ++#endif ++ ++ sd->backlog.poll = process_backlog; ++ sd->backlog.weight = weight_p; ++ } ++ ++ dev_boot_phase = 0; ++ ++ /* The loopback device is special if any other network devices ++ * is present in a network namespace the loopback device must ++ * be present. Since we now dynamically allocate and free the ++ * loopback device ensure this invariant is maintained by ++ * keeping the loopback device as the first device on the ++ * list of network devices. Ensuring the loopback devices ++ * is the first device that appears and the last network device ++ * that disappears. ++ */ ++ if (register_pernet_device(&loopback_net_ops)) ++ goto out; ++ ++ if (register_pernet_device(&default_device_ops)) ++ goto out; ++ ++ open_softirq(NET_TX_SOFTIRQ, net_tx_action); ++ open_softirq(NET_RX_SOFTIRQ, net_rx_action); ++ ++ hotcpu_notifier(dev_cpu_callback, 0); ++ dst_init(); ++ rc = 0; ++out: ++ return rc; ++} ++ ++subsys_initcall(net_dev_init); +diff -Nur linux-4.1.10.orig/net/core/skbuff.c linux-4.1.10/net/core/skbuff.c +--- linux-4.1.10.orig/net/core/skbuff.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/core/skbuff.c 2015-10-07 18:00:08.000000000 +0200 @@ -63,6 +63,7 @@ #include <linux/errqueue.h> #include <linux/prefetch.h> @@ -26299,9 +70088,9 @@ diff -Nur linux-4.1.6.orig/net/core/skbuff.c linux-4.1.6/net/core/skbuff.c return data; } -diff -Nur linux-4.1.6.orig/net/core/sock.c linux-4.1.6/net/core/sock.c ---- linux-4.1.6.orig/net/core/sock.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/core/sock.c 2015-09-08 23:49:08.445829687 +0200 +diff -Nur linux-4.1.10.orig/net/core/sock.c linux-4.1.10/net/core/sock.c +--- linux-4.1.10.orig/net/core/sock.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/core/sock.c 2015-10-07 18:00:08.000000000 +0200 @@ -2370,12 +2370,11 @@ if (sk->sk_lock.owned) __lock_sock(sk); @@ -26316,9 +70105,9 @@ diff -Nur linux-4.1.6.orig/net/core/sock.c linux-4.1.6/net/core/sock.c } EXPORT_SYMBOL(lock_sock_nested); -diff -Nur linux-4.1.6.orig/net/ipv4/icmp.c linux-4.1.6/net/ipv4/icmp.c ---- linux-4.1.6.orig/net/ipv4/icmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/ipv4/icmp.c 2015-09-08 23:49:08.461827913 +0200 +diff -Nur linux-4.1.10.orig/net/ipv4/icmp.c linux-4.1.10/net/ipv4/icmp.c +--- linux-4.1.10.orig/net/ipv4/icmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/ipv4/icmp.c 2015-10-07 18:00:08.000000000 +0200 @@ -69,6 +69,7 @@ #include <linux/jiffies.h> #include <linux/kernel.h> @@ -26370,9 +70159,9 @@ diff -Nur linux-4.1.6.orig/net/ipv4/icmp.c linux-4.1.6/net/ipv4/icmp.c } /* should there be an ICMP stat for ignored echos? */ return true; -diff -Nur linux-4.1.6.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.6/net/ipv4/sysctl_net_ipv4.c ---- linux-4.1.6.orig/net/ipv4/sysctl_net_ipv4.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/ipv4/sysctl_net_ipv4.c 2015-09-08 23:49:08.461827913 +0200 +diff -Nur linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.10/net/ipv4/sysctl_net_ipv4.c +--- linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/ipv4/sysctl_net_ipv4.c 2015-10-07 18:00:08.000000000 +0200 @@ -779,6 +779,13 @@ .proc_handler = proc_dointvec }, @@ -26387,9 +70176,9 @@ diff -Nur linux-4.1.6.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.6/net/ipv4/sysct .procname = "icmp_ignore_bogus_error_responses", .data = &init_net.ipv4.sysctl_icmp_ignore_bogus_error_responses, .maxlen = sizeof(int), -diff -Nur linux-4.1.6.orig/net/mac80211/rx.c linux-4.1.6/net/mac80211/rx.c ---- linux-4.1.6.orig/net/mac80211/rx.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/mac80211/rx.c 2015-09-08 23:49:08.461827913 +0200 +diff -Nur linux-4.1.10.orig/net/mac80211/rx.c linux-4.1.10/net/mac80211/rx.c +--- linux-4.1.10.orig/net/mac80211/rx.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/mac80211/rx.c 2015-10-07 18:00:08.000000000 +0200 @@ -3554,7 +3554,7 @@ struct ieee80211_supported_band *sband; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); @@ -26399,9 +70188,9 @@ diff -Nur linux-4.1.6.orig/net/mac80211/rx.c linux-4.1.6/net/mac80211/rx.c if (WARN_ON(status->band >= IEEE80211_NUM_BANDS)) goto drop; -diff -Nur linux-4.1.6.orig/net/netfilter/core.c linux-4.1.6/net/netfilter/core.c ---- linux-4.1.6.orig/net/netfilter/core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/netfilter/core.c 2015-09-08 23:49:08.461827913 +0200 +diff -Nur linux-4.1.10.orig/net/netfilter/core.c linux-4.1.10/net/netfilter/core.c +--- linux-4.1.10.orig/net/netfilter/core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/netfilter/core.c 2015-10-07 18:00:08.000000000 +0200 @@ -22,11 +22,17 @@ #include <linux/proc_fs.h> #include <linux/mutex.h> @@ -26420,9 +70209,9 @@ diff -Nur linux-4.1.6.orig/net/netfilter/core.c linux-4.1.6/net/netfilter/core.c static DEFINE_MUTEX(afinfo_mutex); const struct nf_afinfo __rcu *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly; -diff -Nur linux-4.1.6.orig/net/packet/af_packet.c linux-4.1.6/net/packet/af_packet.c ---- linux-4.1.6.orig/net/packet/af_packet.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/packet/af_packet.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/net/packet/af_packet.c linux-4.1.10/net/packet/af_packet.c +--- linux-4.1.10.orig/net/packet/af_packet.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/packet/af_packet.c 2015-10-07 18:00:08.000000000 +0200 @@ -63,6 +63,7 @@ #include <linux/if_packet.h> #include <linux/wireless.h> @@ -26449,9 +70238,9 @@ diff -Nur linux-4.1.6.orig/net/packet/af_packet.c linux-4.1.6/net/packet/af_pack } } prb_close_block(pkc, pbd, po, status); -diff -Nur linux-4.1.6.orig/net/rds/ib_rdma.c linux-4.1.6/net/rds/ib_rdma.c ---- linux-4.1.6.orig/net/rds/ib_rdma.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/rds/ib_rdma.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/net/rds/ib_rdma.c linux-4.1.10/net/rds/ib_rdma.c +--- linux-4.1.10.orig/net/rds/ib_rdma.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/rds/ib_rdma.c 2015-10-07 18:00:08.000000000 +0200 @@ -34,6 +34,7 @@ #include <linux/slab.h> #include <linux/rculist.h> @@ -26469,9 +70258,9 @@ diff -Nur linux-4.1.6.orig/net/rds/ib_rdma.c linux-4.1.6/net/rds/ib_rdma.c } } -diff -Nur linux-4.1.6.orig/net/sched/sch_generic.c linux-4.1.6/net/sched/sch_generic.c ---- linux-4.1.6.orig/net/sched/sch_generic.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/sched/sch_generic.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/net/sched/sch_generic.c linux-4.1.10/net/sched/sch_generic.c +--- linux-4.1.10.orig/net/sched/sch_generic.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/sched/sch_generic.c 2015-10-07 18:00:08.000000000 +0200 @@ -894,7 +894,7 @@ /* Wait for outstanding qdisc_run calls. */ list_for_each_entry(dev, head, close_list) @@ -26481,9 +70270,9 @@ diff -Nur linux-4.1.6.orig/net/sched/sch_generic.c linux-4.1.6/net/sched/sch_gen } void dev_deactivate(struct net_device *dev) -diff -Nur linux-4.1.6.orig/net/sunrpc/svc_xprt.c linux-4.1.6/net/sunrpc/svc_xprt.c ---- linux-4.1.6.orig/net/sunrpc/svc_xprt.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/sunrpc/svc_xprt.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/net/sunrpc/svc_xprt.c linux-4.1.10/net/sunrpc/svc_xprt.c +--- linux-4.1.10.orig/net/sunrpc/svc_xprt.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/sunrpc/svc_xprt.c 2015-10-07 18:00:08.000000000 +0200 @@ -341,7 +341,7 @@ goto out; } @@ -26511,9 +70300,9 @@ diff -Nur linux-4.1.6.orig/net/sunrpc/svc_xprt.c linux-4.1.6/net/sunrpc/svc_xprt out: trace_svc_xprt_do_enqueue(xprt, rqstp); } -diff -Nur linux-4.1.6.orig/scripts/mkcompile_h linux-4.1.6/scripts/mkcompile_h ---- linux-4.1.6.orig/scripts/mkcompile_h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/scripts/mkcompile_h 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/scripts/mkcompile_h linux-4.1.10/scripts/mkcompile_h +--- linux-4.1.10.orig/scripts/mkcompile_h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/scripts/mkcompile_h 2015-10-07 18:00:08.000000000 +0200 @@ -4,7 +4,8 @@ ARCH=$2 SMP=$3 @@ -26532,9 +70321,9 @@ diff -Nur linux-4.1.6.orig/scripts/mkcompile_h linux-4.1.6/scripts/mkcompile_h UTS_VERSION="$UTS_VERSION $CONFIG_FLAGS $TIMESTAMP" # Truncate to maximum length -diff -Nur linux-4.1.6.orig/sound/core/pcm_native.c linux-4.1.6/sound/core/pcm_native.c ---- linux-4.1.6.orig/sound/core/pcm_native.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/sound/core/pcm_native.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/sound/core/pcm_native.c linux-4.1.10/sound/core/pcm_native.c +--- linux-4.1.10.orig/sound/core/pcm_native.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/sound/core/pcm_native.c 2015-10-07 18:00:08.000000000 +0200 @@ -123,7 +123,7 @@ void snd_pcm_stream_lock_irq(struct snd_pcm_substream *substream) { @@ -26571,9 +70360,9 @@ diff -Nur linux-4.1.6.orig/sound/core/pcm_native.c linux-4.1.6/sound/core/pcm_na } EXPORT_SYMBOL_GPL(snd_pcm_stream_unlock_irqrestore); -diff -Nur linux-4.1.6.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.6/sound/soc/intel/atom/sst/sst.c ---- linux-4.1.6.orig/sound/soc/intel/atom/sst/sst.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/sound/soc/intel/atom/sst/sst.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.10/sound/soc/intel/atom/sst/sst.c +--- linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/sound/soc/intel/atom/sst/sst.c 2015-10-07 18:00:08.000000000 +0200 @@ -368,8 +368,8 @@ * initialize by FW or driver when firmware is loaded */ @@ -26585,9 +70374,9 @@ diff -Nur linux-4.1.6.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.6/sound/soc/ spin_unlock_irqrestore(&ctx->ipc_spin_lock, irq_flags); } -diff -Nur linux-4.1.6.orig/virt/kvm/async_pf.c linux-4.1.6/virt/kvm/async_pf.c ---- linux-4.1.6.orig/virt/kvm/async_pf.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/virt/kvm/async_pf.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/virt/kvm/async_pf.c linux-4.1.10/virt/kvm/async_pf.c +--- linux-4.1.10.orig/virt/kvm/async_pf.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/virt/kvm/async_pf.c 2015-10-07 18:00:08.000000000 +0200 @@ -94,8 +94,8 @@ trace_kvm_async_pf_completed(addr, gva); @@ -26599,9 +70388,9 @@ diff -Nur linux-4.1.6.orig/virt/kvm/async_pf.c linux-4.1.6/virt/kvm/async_pf.c mmput(mm); kvm_put_kvm(vcpu->kvm); -diff -Nur linux-4.1.6.orig/virt/kvm/kvm_main.c linux-4.1.6/virt/kvm/kvm_main.c ---- linux-4.1.6.orig/virt/kvm/kvm_main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/virt/kvm/kvm_main.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/virt/kvm/kvm_main.c linux-4.1.10/virt/kvm/kvm_main.c +--- linux-4.1.10.orig/virt/kvm/kvm_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/virt/kvm/kvm_main.c 2015-10-07 18:00:08.000000000 +0200 @@ -218,7 +218,7 @@ vcpu->kvm = kvm; vcpu->vcpu_id = id; |