summaryrefslogtreecommitdiff
path: root/target
diff options
context:
space:
mode:
authorWaldemar Brodkorb <wbx@openadk.org>2015-10-12 22:34:38 +0200
committerWaldemar Brodkorb <wbx@openadk.org>2015-10-12 22:35:11 +0200
commitd6f39be098dacce6ca8b2362cf82fd0dcd0b8a6d (patch)
treecfdf9463436ab5947fef8eea4acb948dbb5c8b97 /target
parentd23e37c7c604ebd105bacc3f09afb5546952672b (diff)
cleanup .orig files from patch
Diffstat (limited to 'target')
-rw-r--r--target/linux/patches/4.1.10/realtime.patch44135
1 files changed, 411 insertions, 43724 deletions
diff --git a/target/linux/patches/4.1.10/realtime.patch b/target/linux/patches/4.1.10/realtime.patch
index af7a748cb..fef472a8b 100644
--- a/target/linux/patches/4.1.10/realtime.patch
+++ b/target/linux/patches/4.1.10/realtime.patch
@@ -1,6 +1,6 @@
diff -Nur linux-4.1.10.orig/arch/alpha/mm/fault.c linux-4.1.10/arch/alpha/mm/fault.c
--- linux-4.1.10.orig/arch/alpha/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/alpha/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/alpha/mm/fault.c 2015-10-12 22:33:32.144685475 +0200
@@ -23,8 +23,7 @@
#include <linux/smp.h>
#include <linux/interrupt.h>
@@ -22,7 +22,7 @@ diff -Nur linux-4.1.10.orig/arch/alpha/mm/fault.c linux-4.1.10/arch/alpha/mm/fau
#ifdef CONFIG_ALPHA_LARGE_VMALLOC
diff -Nur linux-4.1.10.orig/arch/arc/include/asm/futex.h linux-4.1.10/arch/arc/include/asm/futex.h
--- linux-4.1.10.orig/arch/arc/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arc/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arc/include/asm/futex.h 2015-10-12 22:33:32.144685475 +0200
@@ -53,7 +53,7 @@
if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
return -EFAULT;
@@ -70,7 +70,7 @@ diff -Nur linux-4.1.10.orig/arch/arc/include/asm/futex.h linux-4.1.10/arch/arc/i
return val;
diff -Nur linux-4.1.10.orig/arch/arc/mm/fault.c linux-4.1.10/arch/arc/mm/fault.c
--- linux-4.1.10.orig/arch/arc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arc/mm/fault.c 2015-10-12 22:33:32.144685475 +0200
@@ -86,7 +86,7 @@
* If we're in an interrupt or have no user
* context, we must not take the fault..
@@ -82,7 +82,7 @@ diff -Nur linux-4.1.10.orig/arch/arc/mm/fault.c linux-4.1.10/arch/arc/mm/fault.c
if (user_mode(regs))
diff -Nur linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.10/arch/arm/include/asm/cmpxchg.h
--- linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/include/asm/cmpxchg.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/include/asm/cmpxchg.h 2015-10-12 22:33:32.148685211 +0200
@@ -129,6 +129,8 @@
#else /* min ARCH >= ARMv6 */
@@ -94,7 +94,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.10/arch/arm
/*
diff -Nur linux-4.1.10.orig/arch/arm/include/asm/futex.h linux-4.1.10/arch/arm/include/asm/futex.h
--- linux-4.1.10.orig/arch/arm/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/include/asm/futex.h 2015-10-12 22:33:32.148685211 +0200
@@ -93,6 +93,7 @@
if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
return -EFAULT;
@@ -138,7 +138,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/include/asm/futex.h linux-4.1.10/arch/arm/i
switch (cmp) {
diff -Nur linux-4.1.10.orig/arch/arm/include/asm/switch_to.h linux-4.1.10/arch/arm/include/asm/switch_to.h
--- linux-4.1.10.orig/arch/arm/include/asm/switch_to.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/include/asm/switch_to.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/include/asm/switch_to.h 2015-10-12 22:33:32.148685211 +0200
@@ -3,6 +3,13 @@
#include <linux/thread_info.h>
@@ -163,7 +163,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/include/asm/switch_to.h linux-4.1.10/arch/a
diff -Nur linux-4.1.10.orig/arch/arm/include/asm/thread_info.h linux-4.1.10/arch/arm/include/asm/thread_info.h
--- linux-4.1.10.orig/arch/arm/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/include/asm/thread_info.h 2015-10-12 22:33:32.148685211 +0200
@@ -50,6 +50,7 @@
struct thread_info {
unsigned long flags; /* low level flags */
@@ -190,7 +190,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/include/asm/thread_info.h linux-4.1.10/arch
#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
diff -Nur linux-4.1.10.orig/arch/arm/Kconfig linux-4.1.10/arch/arm/Kconfig
--- linux-4.1.10.orig/arch/arm/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/Kconfig 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/Kconfig 2015-10-12 22:33:32.148685211 +0200
@@ -31,7 +31,7 @@
select HARDIRQS_SW_RESEND
select HAVE_ARCH_AUDITSYSCALL if (AEABI && !OABI_COMPAT)
@@ -210,7 +210,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/Kconfig linux-4.1.10/arch/arm/Kconfig
select HAVE_SYSCALL_TRACEPOINTS
diff -Nur linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c linux-4.1.10/arch/arm/kernel/asm-offsets.c
--- linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/kernel/asm-offsets.c 2015-10-12 22:33:32.148685211 +0200
@@ -65,6 +65,7 @@
BLANK();
DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
@@ -221,7 +221,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c linux-4.1.10/arch/arm/
DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
diff -Nur linux-4.1.10.orig/arch/arm/kernel/entry-armv.S linux-4.1.10/arch/arm/kernel/entry-armv.S
--- linux-4.1.10.orig/arch/arm/kernel/entry-armv.S 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/kernel/entry-armv.S 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/kernel/entry-armv.S 2015-10-12 22:33:32.152684946 +0200
@@ -208,11 +208,18 @@
#ifdef CONFIG_PREEMPT
get_thread_info tsk
@@ -254,7 +254,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/entry-armv.S linux-4.1.10/arch/arm/k
#endif
diff -Nur linux-4.1.10.orig/arch/arm/kernel/process.c linux-4.1.10/arch/arm/kernel/process.c
--- linux-4.1.10.orig/arch/arm/kernel/process.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/kernel/process.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/kernel/process.c 2015-10-12 22:33:32.152684946 +0200
@@ -290,6 +290,30 @@
}
@@ -288,7 +288,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/process.c linux-4.1.10/arch/arm/kern
* The vectors page is always readable from user space for the
diff -Nur linux-4.1.10.orig/arch/arm/kernel/signal.c linux-4.1.10/arch/arm/kernel/signal.c
--- linux-4.1.10.orig/arch/arm/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/kernel/signal.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/kernel/signal.c 2015-10-12 22:33:32.152684946 +0200
@@ -563,7 +563,8 @@
do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall)
{
@@ -301,7 +301,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/signal.c linux-4.1.10/arch/arm/kerne
if (unlikely(!user_mode(regs)))
diff -Nur linux-4.1.10.orig/arch/arm/kernel/smp.c linux-4.1.10/arch/arm/kernel/smp.c
--- linux-4.1.10.orig/arch/arm/kernel/smp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/kernel/smp.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/kernel/smp.c 2015-10-12 22:33:32.152684946 +0200
@@ -213,8 +213,6 @@
flush_cache_louis();
local_flush_tlb_all();
@@ -323,7 +323,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/smp.c linux-4.1.10/arch/arm/kernel/s
/*
diff -Nur linux-4.1.10.orig/arch/arm/kernel/unwind.c linux-4.1.10/arch/arm/kernel/unwind.c
--- linux-4.1.10.orig/arch/arm/kernel/unwind.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/kernel/unwind.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/kernel/unwind.c 2015-10-12 22:33:32.152684946 +0200
@@ -93,7 +93,7 @@
static const struct unwind_idx *__origin_unwind_idx;
extern const struct unwind_idx __stop_unwind_idx[];
@@ -377,7 +377,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/unwind.c linux-4.1.10/arch/arm/kerne
}
diff -Nur linux-4.1.10.orig/arch/arm/kvm/arm.c linux-4.1.10/arch/arm/kvm/arm.c
--- linux-4.1.10.orig/arch/arm/kvm/arm.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/kvm/arm.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/kvm/arm.c 2015-10-12 22:33:32.156684682 +0200
@@ -474,9 +474,9 @@
static void vcpu_pause(struct kvm_vcpu *vcpu)
@@ -392,7 +392,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kvm/arm.c linux-4.1.10/arch/arm/kvm/arm.c
static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
diff -Nur linux-4.1.10.orig/arch/arm/kvm/psci.c linux-4.1.10/arch/arm/kvm/psci.c
--- linux-4.1.10.orig/arch/arm/kvm/psci.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/kvm/psci.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/kvm/psci.c 2015-10-12 22:33:32.156684682 +0200
@@ -68,7 +68,7 @@
{
struct kvm *kvm = source_vcpu->kvm;
@@ -413,7 +413,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kvm/psci.c linux-4.1.10/arch/arm/kvm/psci.c
}
diff -Nur linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.10/arch/arm/mach-exynos/platsmp.c
--- linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mach-exynos/platsmp.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mach-exynos/platsmp.c 2015-10-12 22:33:32.156684682 +0200
@@ -231,7 +231,7 @@
return (void __iomem *)(S5P_VA_SCU);
}
@@ -463,7 +463,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.10/arch/arm
}
diff -Nur linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.10/arch/arm/mach-hisi/platmcpm.c
--- linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mach-hisi/platmcpm.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mach-hisi/platmcpm.c 2015-10-12 22:33:32.156684682 +0200
@@ -57,7 +57,7 @@
static void __iomem *sysctrl, *fabric;
@@ -568,7 +568,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.10/arch/arm/
static void __naked hip04_mcpm_power_up_setup(unsigned int affinity_level)
diff -Nur linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.10/arch/arm/mach-omap2/omap-smp.c
--- linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mach-omap2/omap-smp.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mach-omap2/omap-smp.c 2015-10-12 22:33:32.156684682 +0200
@@ -43,7 +43,7 @@
/* SCU base address */
static void __iomem *scu_base;
@@ -609,7 +609,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.10/arch/arm
}
diff -Nur linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.10/arch/arm/mach-prima2/platsmp.c
--- linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mach-prima2/platsmp.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mach-prima2/platsmp.c 2015-10-12 22:33:32.156684682 +0200
@@ -22,7 +22,7 @@
static void __iomem *clk_base;
@@ -650,7 +650,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.10/arch/arm
}
diff -Nur linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.10/arch/arm/mach-qcom/platsmp.c
--- linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mach-qcom/platsmp.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mach-qcom/platsmp.c 2015-10-12 22:33:32.160684418 +0200
@@ -46,7 +46,7 @@
extern void secondary_startup_arm(void);
@@ -691,7 +691,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.10/arch/arm/m
}
diff -Nur linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c linux-4.1.10/arch/arm/mach-spear/platsmp.c
--- linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mach-spear/platsmp.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mach-spear/platsmp.c 2015-10-12 22:33:32.160684418 +0200
@@ -32,7 +32,7 @@
sync_cache_w(&pen_release);
}
@@ -732,7 +732,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c linux-4.1.10/arch/arm/
}
diff -Nur linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c linux-4.1.10/arch/arm/mach-sti/platsmp.c
--- linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mach-sti/platsmp.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mach-sti/platsmp.c 2015-10-12 22:33:32.160684418 +0200
@@ -34,7 +34,7 @@
sync_cache_w(&pen_release);
}
@@ -773,7 +773,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c linux-4.1.10/arch/arm/ma
}
diff -Nur linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.10/arch/arm/mach-ux500/platsmp.c
--- linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mach-ux500/platsmp.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mach-ux500/platsmp.c 2015-10-12 22:33:32.160684418 +0200
@@ -51,7 +51,7 @@
return NULL;
}
@@ -814,7 +814,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.10/arch/arm/
}
diff -Nur linux-4.1.10.orig/arch/arm/mm/fault.c linux-4.1.10/arch/arm/mm/fault.c
--- linux-4.1.10.orig/arch/arm/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mm/fault.c 2015-10-12 22:33:32.160684418 +0200
@@ -276,7 +276,7 @@
* If we're in an interrupt or have no user
* context, we must not take the fault..
@@ -846,7 +846,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mm/fault.c linux-4.1.10/arch/arm/mm/fault.c
}
diff -Nur linux-4.1.10.orig/arch/arm/mm/highmem.c linux-4.1.10/arch/arm/mm/highmem.c
--- linux-4.1.10.orig/arch/arm/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/mm/highmem.c 2015-10-12 22:33:32.160684418 +0200
@@ -54,11 +54,13 @@
void *kmap_atomic(struct page *page)
@@ -951,7 +951,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mm/highmem.c linux-4.1.10/arch/arm/mm/highm
+#endif
diff -Nur linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.10/arch/arm/plat-versatile/platsmp.c
--- linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm/plat-versatile/platsmp.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm/plat-versatile/platsmp.c 2015-10-12 22:33:32.160684418 +0200
@@ -30,7 +30,7 @@
sync_cache_w(&pen_release);
}
@@ -992,7 +992,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.10/arch/
}
diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/futex.h linux-4.1.10/arch/arm64/include/asm/futex.h
--- linux-4.1.10.orig/arch/arm64/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm64/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm64/include/asm/futex.h 2015-10-12 22:33:32.160684418 +0200
@@ -58,7 +58,7 @@
if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
return -EFAULT;
@@ -1013,7 +1013,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/futex.h linux-4.1.10/arch/arm
switch (cmp) {
diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h linux-4.1.10/arch/arm64/include/asm/thread_info.h
--- linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm64/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm64/include/asm/thread_info.h 2015-10-12 22:33:32.164684154 +0200
@@ -47,6 +47,7 @@
mm_segment_t addr_limit; /* address limit */
struct task_struct *task; /* main task structure */
@@ -1040,7 +1040,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h linux-4.1.10/ar
#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
diff -Nur linux-4.1.10.orig/arch/arm64/Kconfig linux-4.1.10/arch/arm64/Kconfig
--- linux-4.1.10.orig/arch/arm64/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm64/Kconfig 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm64/Kconfig 2015-10-12 22:33:32.164684154 +0200
@@ -69,8 +69,10 @@
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
@@ -1054,7 +1054,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/Kconfig linux-4.1.10/arch/arm64/Kconfig
select OF
diff -Nur linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.10/arch/arm64/kernel/asm-offsets.c
--- linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm64/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm64/kernel/asm-offsets.c 2015-10-12 22:33:32.164684154 +0200
@@ -35,6 +35,7 @@
BLANK();
DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
@@ -1065,7 +1065,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.10/arch/ar
DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
diff -Nur linux-4.1.10.orig/arch/arm64/kernel/entry.S linux-4.1.10/arch/arm64/kernel/entry.S
--- linux-4.1.10.orig/arch/arm64/kernel/entry.S 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm64/kernel/entry.S 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm64/kernel/entry.S 2015-10-12 22:33:32.164684154 +0200
@@ -367,11 +367,16 @@
#ifdef CONFIG_PREEMPT
get_thread_info tsk
@@ -1104,7 +1104,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/kernel/entry.S linux-4.1.10/arch/arm64/ke
mov x0, sp // 'regs'
diff -Nur linux-4.1.10.orig/arch/arm64/kernel/perf_event.c linux-4.1.10/arch/arm64/kernel/perf_event.c
--- linux-4.1.10.orig/arch/arm64/kernel/perf_event.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm64/kernel/perf_event.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm64/kernel/perf_event.c 2015-10-12 22:33:32.164684154 +0200
@@ -488,7 +488,7 @@
}
@@ -1116,7 +1116,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/kernel/perf_event.c linux-4.1.10/arch/arm
pr_err("unable to request IRQ%d for ARM PMU counters\n",
diff -Nur linux-4.1.10.orig/arch/arm64/mm/fault.c linux-4.1.10/arch/arm64/mm/fault.c
--- linux-4.1.10.orig/arch/arm64/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/arm64/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/arm64/mm/fault.c 2015-10-12 22:33:32.168683889 +0200
@@ -211,7 +211,7 @@
* If we're in an interrupt or have no user context, we must not take
* the fault.
@@ -1128,7 +1128,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/mm/fault.c linux-4.1.10/arch/arm64/mm/fau
if (user_mode(regs))
diff -Nur linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h linux-4.1.10/arch/avr32/include/asm/uaccess.h
--- linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/avr32/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/avr32/include/asm/uaccess.h 2015-10-12 22:33:32.180683097 +0200
@@ -97,7 +97,8 @@
* @x: Value to copy to user space.
* @ptr: Destination address, in user space.
@@ -1171,7 +1171,7 @@ diff -Nur linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h linux-4.1.10/arch/a
* space. It supports simple types like char and int, but not larger
diff -Nur linux-4.1.10.orig/arch/avr32/mm/fault.c linux-4.1.10/arch/avr32/mm/fault.c
--- linux-4.1.10.orig/arch/avr32/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/avr32/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/avr32/mm/fault.c 2015-10-12 22:33:32.180683097 +0200
@@ -14,11 +14,11 @@
#include <linux/pagemap.h>
#include <linux/kdebug.h>
@@ -1196,7 +1196,7 @@ diff -Nur linux-4.1.10.orig/arch/avr32/mm/fault.c linux-4.1.10/arch/avr32/mm/fau
local_irq_enable();
diff -Nur linux-4.1.10.orig/arch/cris/mm/fault.c linux-4.1.10/arch/cris/mm/fault.c
--- linux-4.1.10.orig/arch/cris/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/cris/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/cris/mm/fault.c 2015-10-12 22:33:32.180683097 +0200
@@ -8,7 +8,7 @@
#include <linux/interrupt.h>
#include <linux/module.h>
@@ -1222,7 +1222,7 @@ diff -Nur linux-4.1.10.orig/arch/cris/mm/fault.c linux-4.1.10/arch/cris/mm/fault
if (user_mode(regs))
diff -Nur linux-4.1.10.orig/arch/frv/mm/fault.c linux-4.1.10/arch/frv/mm/fault.c
--- linux-4.1.10.orig/arch/frv/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/frv/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/frv/mm/fault.c 2015-10-12 22:33:32.180683097 +0200
@@ -19,9 +19,9 @@
#include <linux/kernel.h>
#include <linux/ptrace.h>
@@ -1245,7 +1245,7 @@ diff -Nur linux-4.1.10.orig/arch/frv/mm/fault.c linux-4.1.10/arch/frv/mm/fault.c
if (user_mode(__frame))
diff -Nur linux-4.1.10.orig/arch/frv/mm/highmem.c linux-4.1.10/arch/frv/mm/highmem.c
--- linux-4.1.10.orig/arch/frv/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/frv/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/frv/mm/highmem.c 2015-10-12 22:33:32.180683097 +0200
@@ -42,6 +42,7 @@
unsigned long paddr;
int type;
@@ -1263,7 +1263,7 @@ diff -Nur linux-4.1.10.orig/arch/frv/mm/highmem.c linux-4.1.10/arch/frv/mm/highm
EXPORT_SYMBOL(__kunmap_atomic);
diff -Nur linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.10/arch/hexagon/include/asm/uaccess.h
--- linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/hexagon/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/hexagon/include/asm/uaccess.h 2015-10-12 22:33:32.180683097 +0200
@@ -36,7 +36,8 @@
* @addr: User space pointer to start of block to check
* @size: Size of block to check
@@ -1276,7 +1276,7 @@ diff -Nur linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.10/arch
*
diff -Nur linux-4.1.10.orig/arch/ia64/mm/fault.c linux-4.1.10/arch/ia64/mm/fault.c
--- linux-4.1.10.orig/arch/ia64/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/ia64/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/ia64/mm/fault.c 2015-10-12 22:33:32.184682832 +0200
@@ -11,10 +11,10 @@
#include <linux/kprobes.h>
#include <linux/kdebug.h>
@@ -1300,7 +1300,7 @@ diff -Nur linux-4.1.10.orig/arch/ia64/mm/fault.c linux-4.1.10/arch/ia64/mm/fault
#ifdef CONFIG_VIRTUAL_MEM_MAP
diff -Nur linux-4.1.10.orig/arch/Kconfig linux-4.1.10/arch/Kconfig
--- linux-4.1.10.orig/arch/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/Kconfig 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/Kconfig 2015-10-12 22:33:32.184682832 +0200
@@ -6,6 +6,7 @@
tristate "OProfile system profiling"
depends on PROFILING
@@ -1311,7 +1311,7 @@ diff -Nur linux-4.1.10.orig/arch/Kconfig linux-4.1.10/arch/Kconfig
help
diff -Nur linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h linux-4.1.10/arch/m32r/include/asm/uaccess.h
--- linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/m32r/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/m32r/include/asm/uaccess.h 2015-10-12 22:33:32.184682832 +0200
@@ -91,7 +91,8 @@
* @addr: User space pointer to start of block to check
* @size: Size of block to check
@@ -1414,7 +1414,7 @@ diff -Nur linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h linux-4.1.10/arch/m3
*
diff -Nur linux-4.1.10.orig/arch/m32r/mm/fault.c linux-4.1.10/arch/m32r/mm/fault.c
--- linux-4.1.10.orig/arch/m32r/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/m32r/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/m32r/mm/fault.c 2015-10-12 22:33:32.184682832 +0200
@@ -24,9 +24,9 @@
#include <linux/vt_kern.h> /* For unblank_screen() */
#include <linux/highmem.h>
@@ -1442,7 +1442,7 @@ diff -Nur linux-4.1.10.orig/arch/m32r/mm/fault.c linux-4.1.10/arch/m32r/mm/fault
if (error_code & ACE_USERMODE)
diff -Nur linux-4.1.10.orig/arch/m68k/mm/fault.c linux-4.1.10/arch/m68k/mm/fault.c
--- linux-4.1.10.orig/arch/m68k/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/m68k/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/m68k/mm/fault.c 2015-10-12 22:33:32.184682832 +0200
@@ -10,10 +10,10 @@
#include <linux/ptrace.h>
#include <linux/interrupt.h>
@@ -1466,7 +1466,7 @@ diff -Nur linux-4.1.10.orig/arch/m68k/mm/fault.c linux-4.1.10/arch/m68k/mm/fault
if (user_mode(regs))
diff -Nur linux-4.1.10.orig/arch/metag/mm/fault.c linux-4.1.10/arch/metag/mm/fault.c
--- linux-4.1.10.orig/arch/metag/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/metag/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/metag/mm/fault.c 2015-10-12 22:33:32.184682832 +0200
@@ -105,7 +105,7 @@
mm = tsk->mm;
@@ -1478,7 +1478,7 @@ diff -Nur linux-4.1.10.orig/arch/metag/mm/fault.c linux-4.1.10/arch/metag/mm/fau
if (user_mode(regs))
diff -Nur linux-4.1.10.orig/arch/metag/mm/highmem.c linux-4.1.10/arch/metag/mm/highmem.c
--- linux-4.1.10.orig/arch/metag/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/metag/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/metag/mm/highmem.c 2015-10-12 22:33:32.184682832 +0200
@@ -43,7 +43,7 @@
unsigned long vaddr;
int type;
@@ -1506,7 +1506,7 @@ diff -Nur linux-4.1.10.orig/arch/metag/mm/highmem.c linux-4.1.10/arch/metag/mm/h
type = kmap_atomic_idx_push();
diff -Nur linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.10/arch/microblaze/include/asm/uaccess.h
--- linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/microblaze/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/microblaze/include/asm/uaccess.h 2015-10-12 22:33:32.188682568 +0200
@@ -178,7 +178,8 @@
* @x: Variable to store result.
* @ptr: Source address, in user space.
@@ -1529,7 +1529,7 @@ diff -Nur linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.10/a
* space. It supports simple types like char and int, but not larger
diff -Nur linux-4.1.10.orig/arch/microblaze/mm/fault.c linux-4.1.10/arch/microblaze/mm/fault.c
--- linux-4.1.10.orig/arch/microblaze/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/microblaze/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/microblaze/mm/fault.c 2015-10-12 22:33:32.188682568 +0200
@@ -107,14 +107,14 @@
if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11)
is_write = 0;
@@ -1551,7 +1551,7 @@ diff -Nur linux-4.1.10.orig/arch/microblaze/mm/fault.c linux-4.1.10/arch/microbl
die("Weird page fault", regs, SIGSEGV);
diff -Nur linux-4.1.10.orig/arch/microblaze/mm/highmem.c linux-4.1.10/arch/microblaze/mm/highmem.c
--- linux-4.1.10.orig/arch/microblaze/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/microblaze/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/microblaze/mm/highmem.c 2015-10-12 22:33:32.188682568 +0200
@@ -37,7 +37,7 @@
unsigned long vaddr;
int idx, type;
@@ -1578,7 +1578,7 @@ diff -Nur linux-4.1.10.orig/arch/microblaze/mm/highmem.c linux-4.1.10/arch/micro
EXPORT_SYMBOL(__kunmap_atomic);
diff -Nur linux-4.1.10.orig/arch/mips/include/asm/uaccess.h linux-4.1.10/arch/mips/include/asm/uaccess.h
--- linux-4.1.10.orig/arch/mips/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/mips/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/mips/include/asm/uaccess.h 2015-10-12 22:33:32.188682568 +0200
@@ -103,7 +103,8 @@
* @addr: User space pointer to start of block to check
* @size: Size of block to check
@@ -1731,7 +1731,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/include/asm/uaccess.h linux-4.1.10/arch/mi
*
diff -Nur linux-4.1.10.orig/arch/mips/Kconfig linux-4.1.10/arch/mips/Kconfig
--- linux-4.1.10.orig/arch/mips/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/mips/Kconfig 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/mips/Kconfig 2015-10-12 22:33:32.192682304 +0200
@@ -2366,7 +2366,7 @@
#
config HIGHMEM
@@ -1743,7 +1743,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/Kconfig linux-4.1.10/arch/mips/Kconfig
bool
diff -Nur linux-4.1.10.orig/arch/mips/kernel/signal-common.h linux-4.1.10/arch/mips/kernel/signal-common.h
--- linux-4.1.10.orig/arch/mips/kernel/signal-common.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/mips/kernel/signal-common.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/mips/kernel/signal-common.h 2015-10-12 22:33:32.192682304 +0200
@@ -28,12 +28,7 @@
extern int fpcsr_pending(unsigned int __user *fpcsr);
@@ -1761,7 +1761,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/kernel/signal-common.h linux-4.1.10/arch/m
#endif /* __SIGNAL_COMMON_H */
diff -Nur linux-4.1.10.orig/arch/mips/mm/fault.c linux-4.1.10/arch/mips/mm/fault.c
--- linux-4.1.10.orig/arch/mips/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/mips/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/mips/mm/fault.c 2015-10-12 22:33:32.192682304 +0200
@@ -21,10 +21,10 @@
#include <linux/module.h>
#include <linux/kprobes.h>
@@ -1785,7 +1785,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/mm/fault.c linux-4.1.10/arch/mips/mm/fault
if (user_mode(regs))
diff -Nur linux-4.1.10.orig/arch/mips/mm/highmem.c linux-4.1.10/arch/mips/mm/highmem.c
--- linux-4.1.10.orig/arch/mips/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/mips/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/mips/mm/highmem.c 2015-10-12 22:33:32.192682304 +0200
@@ -47,7 +47,7 @@
unsigned long vaddr;
int idx, type;
@@ -1821,7 +1821,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/mm/highmem.c linux-4.1.10/arch/mips/mm/hig
type = kmap_atomic_idx_push();
diff -Nur linux-4.1.10.orig/arch/mips/mm/init.c linux-4.1.10/arch/mips/mm/init.c
--- linux-4.1.10.orig/arch/mips/mm/init.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/mips/mm/init.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/mips/mm/init.c 2015-10-12 22:33:32.192682304 +0200
@@ -90,6 +90,7 @@
BUG_ON(Page_dcache_dirty(page));
@@ -1840,7 +1840,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/mm/init.c linux-4.1.10/arch/mips/mm/init.c
void copy_user_highpage(struct page *to, struct page *from,
diff -Nur linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h linux-4.1.10/arch/mn10300/include/asm/highmem.h
--- linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/mn10300/include/asm/highmem.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/mn10300/include/asm/highmem.h 2015-10-12 22:33:32.192682304 +0200
@@ -75,6 +75,7 @@
unsigned long vaddr;
int idx, type;
@@ -1867,7 +1867,7 @@ diff -Nur linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h linux-4.1.10/arch
diff -Nur linux-4.1.10.orig/arch/mn10300/mm/fault.c linux-4.1.10/arch/mn10300/mm/fault.c
--- linux-4.1.10.orig/arch/mn10300/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/mn10300/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/mn10300/mm/fault.c 2015-10-12 22:33:32.192682304 +0200
@@ -23,8 +23,8 @@
#include <linux/interrupt.h>
#include <linux/init.h>
@@ -1889,7 +1889,7 @@ diff -Nur linux-4.1.10.orig/arch/mn10300/mm/fault.c linux-4.1.10/arch/mn10300/mm
if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR)
diff -Nur linux-4.1.10.orig/arch/nios2/mm/fault.c linux-4.1.10/arch/nios2/mm/fault.c
--- linux-4.1.10.orig/arch/nios2/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/nios2/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/nios2/mm/fault.c 2015-10-12 22:33:32.192682304 +0200
@@ -77,7 +77,7 @@
* If we're in an interrupt or have no user
* context, we must not take the fault..
@@ -1901,7 +1901,7 @@ diff -Nur linux-4.1.10.orig/arch/nios2/mm/fault.c linux-4.1.10/arch/nios2/mm/fau
if (user_mode(regs))
diff -Nur linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.10/arch/parisc/include/asm/cacheflush.h
--- linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/parisc/include/asm/cacheflush.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/parisc/include/asm/cacheflush.h 2015-10-12 22:33:32.192682304 +0200
@@ -142,6 +142,7 @@
static inline void *kmap_atomic(struct page *page)
@@ -1920,7 +1920,7 @@ diff -Nur linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.10/ar
#define kmap_atomic_prot(page, prot) kmap_atomic(page)
diff -Nur linux-4.1.10.orig/arch/parisc/kernel/traps.c linux-4.1.10/arch/parisc/kernel/traps.c
--- linux-4.1.10.orig/arch/parisc/kernel/traps.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/parisc/kernel/traps.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/parisc/kernel/traps.c 2015-10-12 22:33:32.196682039 +0200
@@ -26,9 +26,9 @@
#include <linux/console.h>
#include <linux/bug.h>
@@ -1943,7 +1943,7 @@ diff -Nur linux-4.1.10.orig/arch/parisc/kernel/traps.c linux-4.1.10/arch/parisc/
parisc_terminate("Kernel Fault", regs, code, fault_address);
diff -Nur linux-4.1.10.orig/arch/parisc/mm/fault.c linux-4.1.10/arch/parisc/mm/fault.c
--- linux-4.1.10.orig/arch/parisc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/parisc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/parisc/mm/fault.c 2015-10-12 22:33:32.196682039 +0200
@@ -15,8 +15,8 @@
#include <linux/sched.h>
#include <linux/interrupt.h>
@@ -1965,7 +1965,7 @@ diff -Nur linux-4.1.10.orig/arch/parisc/mm/fault.c linux-4.1.10/arch/parisc/mm/f
tsk = current;
diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.10/arch/powerpc/include/asm/kvm_host.h
--- linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/include/asm/kvm_host.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/include/asm/kvm_host.h 2015-10-12 22:33:32.196682039 +0200
@@ -280,7 +280,7 @@
u8 in_guest;
struct list_head runnable_threads;
@@ -1986,7 +1986,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.10/arc
int trap;
diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.10/arch/powerpc/include/asm/thread_info.h
--- linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/include/asm/thread_info.h 2015-10-12 22:33:32.196682039 +0200
@@ -42,6 +42,8 @@
int cpu; /* cpu we're on */
int preempt_count; /* 0 => preemptable,
@@ -2035,7 +2035,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.10/
/* Don't move TLF_NAPPING without adjusting the code in entry_32.S */
diff -Nur linux-4.1.10.orig/arch/powerpc/Kconfig linux-4.1.10/arch/powerpc/Kconfig
--- linux-4.1.10.orig/arch/powerpc/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/Kconfig 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/Kconfig 2015-10-12 22:33:32.196682039 +0200
@@ -60,10 +60,11 @@
config RWSEM_GENERIC_SPINLOCK
@@ -2068,7 +2068,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/Kconfig linux-4.1.10/arch/powerpc/Kconf
source kernel/Kconfig.preempt
diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.10/arch/powerpc/kernel/asm-offsets.c
--- linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/kernel/asm-offsets.c 2015-10-12 22:33:32.196682039 +0200
@@ -160,6 +160,7 @@
DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags));
@@ -2079,7 +2079,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.10/arch/
diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S linux-4.1.10/arch/powerpc/kernel/entry_32.S
--- linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/kernel/entry_32.S 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/kernel/entry_32.S 2015-10-12 22:33:32.196682039 +0200
@@ -813,7 +813,14 @@
cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
bne restore
@@ -2130,7 +2130,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S linux-4.1.10/arch/pow
beq restore_user
diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S linux-4.1.10/arch/powerpc/kernel/entry_64.S
--- linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/kernel/entry_64.S 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/kernel/entry_64.S 2015-10-12 22:33:32.196682039 +0200
@@ -636,7 +636,7 @@
#else
beq restore
@@ -2171,7 +2171,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S linux-4.1.10/arch/pow
/*
diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/irq.c linux-4.1.10/arch/powerpc/kernel/irq.c
--- linux-4.1.10.orig/arch/powerpc/kernel/irq.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/kernel/irq.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/kernel/irq.c 2015-10-12 22:33:32.196682039 +0200
@@ -614,6 +614,7 @@
}
}
@@ -2190,7 +2190,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/irq.c linux-4.1.10/arch/powerpc/
{
diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S linux-4.1.10/arch/powerpc/kernel/misc_32.S
--- linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/kernel/misc_32.S 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/kernel/misc_32.S 2015-10-12 22:33:32.196682039 +0200
@@ -40,6 +40,7 @@
* We store the saved ksp_limit in the unused part
* of the STACK_FRAME_OVERHEAD
@@ -2209,7 +2209,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S linux-4.1.10/arch/powe
* void call_do_irq(struct pt_regs *regs, struct thread_info *irqtp);
diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S linux-4.1.10/arch/powerpc/kernel/misc_64.S
--- linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/kernel/misc_64.S 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/kernel/misc_64.S 2015-10-12 22:33:32.208681247 +0200
@@ -29,6 +29,7 @@
.text
@@ -2228,7 +2228,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S linux-4.1.10/arch/powe
mflr r0
diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.10/arch/powerpc/kvm/book3s_hv.c
--- linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/kvm/book3s_hv.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/kvm/book3s_hv.c 2015-10-12 22:33:32.208681247 +0200
@@ -115,11 +115,11 @@
static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
{
@@ -2306,7 +2306,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.10/arch/power
}
diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/Kconfig linux-4.1.10/arch/powerpc/kvm/Kconfig
--- linux-4.1.10.orig/arch/powerpc/kvm/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/kvm/Kconfig 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/kvm/Kconfig 2015-10-12 22:33:32.208681247 +0200
@@ -172,6 +172,7 @@
config KVM_MPIC
bool "KVM in-kernel MPIC emulation"
@@ -2317,7 +2317,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/Kconfig linux-4.1.10/arch/powerpc/k
select HAVE_KVM_IRQ_ROUTING
diff -Nur linux-4.1.10.orig/arch/powerpc/mm/fault.c linux-4.1.10/arch/powerpc/mm/fault.c
--- linux-4.1.10.orig/arch/powerpc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/mm/fault.c 2015-10-12 22:33:32.212680982 +0200
@@ -33,13 +33,13 @@
#include <linux/ratelimit.h>
#include <linux/context_tracking.h>
@@ -2355,7 +2355,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/mm/fault.c linux-4.1.10/arch/powerpc/mm
die("Weird page fault", regs, SIGSEGV);
diff -Nur linux-4.1.10.orig/arch/powerpc/mm/highmem.c linux-4.1.10/arch/powerpc/mm/highmem.c
--- linux-4.1.10.orig/arch/powerpc/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/mm/highmem.c 2015-10-12 22:33:32.212680982 +0200
@@ -34,7 +34,7 @@
unsigned long vaddr;
int idx, type;
@@ -2382,7 +2382,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/mm/highmem.c linux-4.1.10/arch/powerpc/
EXPORT_SYMBOL(__kunmap_atomic);
diff -Nur linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c
--- linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c 2015-10-12 22:33:32.212680982 +0200
@@ -752,7 +752,7 @@
}
pr_debug("%s:%u: notification %s issued\n", __func__, __LINE__, op);
@@ -2394,7 +2394,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.1
res = -EINTR;
diff -Nur linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h linux-4.1.10/arch/s390/include/asm/kvm_host.h
--- linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/s390/include/asm/kvm_host.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/s390/include/asm/kvm_host.h 2015-10-12 22:33:32.212680982 +0200
@@ -419,7 +419,7 @@
struct kvm_s390_local_interrupt {
spinlock_t lock;
@@ -2406,7 +2406,7 @@ diff -Nur linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h linux-4.1.10/arch/s
struct kvm_s390_irq_payload irq;
diff -Nur linux-4.1.10.orig/arch/s390/include/asm/uaccess.h linux-4.1.10/arch/s390/include/asm/uaccess.h
--- linux-4.1.10.orig/arch/s390/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/s390/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/s390/include/asm/uaccess.h 2015-10-12 22:33:32.212680982 +0200
@@ -98,7 +98,8 @@
* @from: Source address, in user space.
* @n: Number of bytes to copy.
@@ -2459,7 +2459,7 @@ diff -Nur linux-4.1.10.orig/arch/s390/include/asm/uaccess.h linux-4.1.10/arch/s3
*
diff -Nur linux-4.1.10.orig/arch/s390/kvm/interrupt.c linux-4.1.10/arch/s390/kvm/interrupt.c
--- linux-4.1.10.orig/arch/s390/kvm/interrupt.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/s390/kvm/interrupt.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/s390/kvm/interrupt.c 2015-10-12 22:33:32.212680982 +0200
@@ -875,13 +875,13 @@
void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
@@ -2496,7 +2496,7 @@ diff -Nur linux-4.1.10.orig/arch/s390/kvm/interrupt.c linux-4.1.10/arch/s390/kvm
}
diff -Nur linux-4.1.10.orig/arch/s390/mm/fault.c linux-4.1.10/arch/s390/mm/fault.c
--- linux-4.1.10.orig/arch/s390/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/s390/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/s390/mm/fault.c 2015-10-12 22:33:32.212680982 +0200
@@ -399,7 +399,7 @@
* user context.
*/
@@ -2508,7 +2508,7 @@ diff -Nur linux-4.1.10.orig/arch/s390/mm/fault.c linux-4.1.10/arch/s390/mm/fault
address = trans_exc_code & __FAIL_ADDR_MASK;
diff -Nur linux-4.1.10.orig/arch/score/include/asm/uaccess.h linux-4.1.10/arch/score/include/asm/uaccess.h
--- linux-4.1.10.orig/arch/score/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/score/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/score/include/asm/uaccess.h 2015-10-12 22:33:32.212680982 +0200
@@ -36,7 +36,8 @@
* @addr: User space pointer to start of block to check
* @size: Size of block to check
@@ -2561,7 +2561,7 @@ diff -Nur linux-4.1.10.orig/arch/score/include/asm/uaccess.h linux-4.1.10/arch/s
* space. It supports simple types like char and int, but not larger
diff -Nur linux-4.1.10.orig/arch/score/mm/fault.c linux-4.1.10/arch/score/mm/fault.c
--- linux-4.1.10.orig/arch/score/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/score/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/score/mm/fault.c 2015-10-12 22:33:32.212680982 +0200
@@ -34,6 +34,7 @@
#include <linux/string.h>
#include <linux/types.h>
@@ -2581,7 +2581,7 @@ diff -Nur linux-4.1.10.orig/arch/score/mm/fault.c linux-4.1.10/arch/score/mm/fau
if (user_mode(regs))
diff -Nur linux-4.1.10.orig/arch/sh/kernel/irq.c linux-4.1.10/arch/sh/kernel/irq.c
--- linux-4.1.10.orig/arch/sh/kernel/irq.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/sh/kernel/irq.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/sh/kernel/irq.c 2015-10-12 22:33:32.212680982 +0200
@@ -147,6 +147,7 @@
hardirq_ctx[cpu] = NULL;
}
@@ -2600,7 +2600,7 @@ diff -Nur linux-4.1.10.orig/arch/sh/kernel/irq.c linux-4.1.10/arch/sh/kernel/irq
{
diff -Nur linux-4.1.10.orig/arch/sh/mm/fault.c linux-4.1.10/arch/sh/mm/fault.c
--- linux-4.1.10.orig/arch/sh/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/sh/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/sh/mm/fault.c 2015-10-12 22:33:32.212680982 +0200
@@ -17,6 +17,7 @@
#include <linux/kprobes.h>
#include <linux/perf_event.h>
@@ -2623,7 +2623,7 @@ diff -Nur linux-4.1.10.orig/arch/sh/mm/fault.c linux-4.1.10/arch/sh/mm/fault.c
}
diff -Nur linux-4.1.10.orig/arch/sparc/Kconfig linux-4.1.10/arch/sparc/Kconfig
--- linux-4.1.10.orig/arch/sparc/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/sparc/Kconfig 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/sparc/Kconfig 2015-10-12 22:33:32.212680982 +0200
@@ -189,12 +189,10 @@
source kernel/Kconfig.hz
@@ -2641,7 +2641,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/Kconfig linux-4.1.10/arch/sparc/Kconfig
bool
diff -Nur linux-4.1.10.orig/arch/sparc/kernel/irq_64.c linux-4.1.10/arch/sparc/kernel/irq_64.c
--- linux-4.1.10.orig/arch/sparc/kernel/irq_64.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/sparc/kernel/irq_64.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/sparc/kernel/irq_64.c 2015-10-12 22:33:32.212680982 +0200
@@ -849,6 +849,7 @@
set_irq_regs(old_regs);
}
@@ -2660,7 +2660,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/kernel/irq_64.c linux-4.1.10/arch/sparc/k
void fixup_irqs(void)
diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_32.c linux-4.1.10/arch/sparc/mm/fault_32.c
--- linux-4.1.10.orig/arch/sparc/mm/fault_32.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/sparc/mm/fault_32.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/sparc/mm/fault_32.c 2015-10-12 22:33:32.212680982 +0200
@@ -21,6 +21,7 @@
#include <linux/perf_event.h>
#include <linux/interrupt.h>
@@ -2688,7 +2688,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_32.c linux-4.1.10/arch/sparc/mm/
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_64.c linux-4.1.10/arch/sparc/mm/fault_64.c
--- linux-4.1.10.orig/arch/sparc/mm/fault_64.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/sparc/mm/fault_64.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/sparc/mm/fault_64.c 2015-10-12 22:33:32.212680982 +0200
@@ -22,12 +22,12 @@
#include <linux/kdebug.h>
#include <linux/percpu.h>
@@ -2714,7 +2714,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_64.c linux-4.1.10/arch/sparc/mm/
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
diff -Nur linux-4.1.10.orig/arch/sparc/mm/highmem.c linux-4.1.10/arch/sparc/mm/highmem.c
--- linux-4.1.10.orig/arch/sparc/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/sparc/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/sparc/mm/highmem.c 2015-10-12 22:33:32.216680718 +0200
@@ -53,7 +53,7 @@
unsigned long vaddr;
long idx, type;
@@ -2741,7 +2741,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/mm/highmem.c linux-4.1.10/arch/sparc/mm/h
EXPORT_SYMBOL(__kunmap_atomic);
diff -Nur linux-4.1.10.orig/arch/sparc/mm/init_64.c linux-4.1.10/arch/sparc/mm/init_64.c
--- linux-4.1.10.orig/arch/sparc/mm/init_64.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/sparc/mm/init_64.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/sparc/mm/init_64.c 2015-10-12 22:33:32.216680718 +0200
@@ -2738,7 +2738,7 @@
struct mm_struct *mm = current->mm;
struct tsb_config *tp;
@@ -2753,7 +2753,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/mm/init_64.c linux-4.1.10/arch/sparc/mm/i
entry = search_exception_tables(regs->tpc);
diff -Nur linux-4.1.10.orig/arch/tile/include/asm/uaccess.h linux-4.1.10/arch/tile/include/asm/uaccess.h
--- linux-4.1.10.orig/arch/tile/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/tile/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/tile/include/asm/uaccess.h 2015-10-12 22:33:32.216680718 +0200
@@ -78,7 +78,8 @@
* @addr: User space pointer to start of block to check
* @size: Size of block to check
@@ -2816,7 +2816,7 @@ diff -Nur linux-4.1.10.orig/arch/tile/include/asm/uaccess.h linux-4.1.10/arch/ti
* the specified blocks with access_ok() before calling this function.
diff -Nur linux-4.1.10.orig/arch/tile/mm/fault.c linux-4.1.10/arch/tile/mm/fault.c
--- linux-4.1.10.orig/arch/tile/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/tile/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/tile/mm/fault.c 2015-10-12 22:33:32.216680718 +0200
@@ -354,9 +354,9 @@
/*
@@ -2831,7 +2831,7 @@ diff -Nur linux-4.1.10.orig/arch/tile/mm/fault.c linux-4.1.10/arch/tile/mm/fault
}
diff -Nur linux-4.1.10.orig/arch/tile/mm/highmem.c linux-4.1.10/arch/tile/mm/highmem.c
--- linux-4.1.10.orig/arch/tile/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/tile/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/tile/mm/highmem.c 2015-10-12 22:33:32.216680718 +0200
@@ -201,7 +201,7 @@
int idx, type;
pte_t *pte;
@@ -2851,7 +2851,7 @@ diff -Nur linux-4.1.10.orig/arch/tile/mm/highmem.c linux-4.1.10/arch/tile/mm/hig
diff -Nur linux-4.1.10.orig/arch/um/kernel/trap.c linux-4.1.10/arch/um/kernel/trap.c
--- linux-4.1.10.orig/arch/um/kernel/trap.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/um/kernel/trap.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/um/kernel/trap.c 2015-10-12 22:33:32.216680718 +0200
@@ -35,10 +35,10 @@
*code_out = SEGV_MAPERR;
@@ -2867,7 +2867,7 @@ diff -Nur linux-4.1.10.orig/arch/um/kernel/trap.c linux-4.1.10/arch/um/kernel/tr
if (is_user)
diff -Nur linux-4.1.10.orig/arch/unicore32/mm/fault.c linux-4.1.10/arch/unicore32/mm/fault.c
--- linux-4.1.10.orig/arch/unicore32/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/unicore32/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/unicore32/mm/fault.c 2015-10-12 22:33:32.216680718 +0200
@@ -218,7 +218,7 @@
* If we're in an interrupt or have no user
* context, we must not take the fault..
@@ -2879,7 +2879,7 @@ diff -Nur linux-4.1.10.orig/arch/unicore32/mm/fault.c linux-4.1.10/arch/unicore3
if (user_mode(regs))
diff -Nur linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c
--- linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c 2015-10-12 22:33:32.216680718 +0200
@@ -382,14 +382,14 @@
err = blkcipher_walk_virt(desc, &walk);
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
@@ -2974,7 +2974,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.10/arch
}
diff -Nur linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c
--- linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c 2015-10-12 22:33:32.216680718 +0200
@@ -60,7 +60,7 @@
static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
bool enc)
@@ -3056,7 +3056,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.10/arch/x
err = blkcipher_walk_done(desc, &walk, 0);
diff -Nur linux-4.1.10.orig/arch/x86/crypto/glue_helper.c linux-4.1.10/arch/x86/crypto/glue_helper.c
--- linux-4.1.10.orig/arch/x86/crypto/glue_helper.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/crypto/glue_helper.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/crypto/glue_helper.c 2015-10-12 22:33:32.216680718 +0200
@@ -39,7 +39,7 @@
void *ctx = crypto_blkcipher_ctx(desc->tfm);
const unsigned int bsize = 128 / 8;
@@ -3174,7 +3174,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/crypto/glue_helper.c linux-4.1.10/arch/x86/
EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit);
diff -Nur linux-4.1.10.orig/arch/x86/include/asm/preempt.h linux-4.1.10/arch/x86/include/asm/preempt.h
--- linux-4.1.10.orig/arch/x86/include/asm/preempt.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/include/asm/preempt.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/include/asm/preempt.h 2015-10-12 22:33:32.216680718 +0200
@@ -82,17 +82,33 @@
* a decrement which hits zero means we have no preempt_count and should
* reschedule.
@@ -3212,7 +3212,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/preempt.h linux-4.1.10/arch/x86
#ifdef CONFIG_PREEMPT
diff -Nur linux-4.1.10.orig/arch/x86/include/asm/signal.h linux-4.1.10/arch/x86/include/asm/signal.h
--- linux-4.1.10.orig/arch/x86/include/asm/signal.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/include/asm/signal.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/include/asm/signal.h 2015-10-12 22:33:32.216680718 +0200
@@ -23,6 +23,19 @@
unsigned long sig[_NSIG_WORDS];
} sigset_t;
@@ -3235,7 +3235,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/signal.h linux-4.1.10/arch/x86/
#endif
diff -Nur linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h linux-4.1.10/arch/x86/include/asm/stackprotector.h
--- linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/include/asm/stackprotector.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/include/asm/stackprotector.h 2015-10-12 22:33:32.216680718 +0200
@@ -57,7 +57,7 @@
*/
static __always_inline void boot_init_stack_canary(void)
@@ -3264,7 +3264,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h linux-4.1.10/a
diff -Nur linux-4.1.10.orig/arch/x86/include/asm/thread_info.h linux-4.1.10/arch/x86/include/asm/thread_info.h
--- linux-4.1.10.orig/arch/x86/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/include/asm/thread_info.h 2015-10-12 22:33:32.220680454 +0200
@@ -55,6 +55,8 @@
__u32 status; /* thread synchronous flags */
__u32 cpu; /* current CPU */
@@ -3301,7 +3301,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/thread_info.h linux-4.1.10/arch
/*
diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.10/arch/x86/include/asm/uaccess_32.h
--- linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/include/asm/uaccess_32.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/include/asm/uaccess_32.h 2015-10-12 22:33:32.220680454 +0200
@@ -70,7 +70,8 @@
* @from: Source address, in kernel space.
* @n: Number of bytes to copy.
@@ -3324,7 +3324,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.10/arch/
* the specified block with access_ok() before calling this function.
diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess.h linux-4.1.10/arch/x86/include/asm/uaccess.h
--- linux-4.1.10.orig/arch/x86/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/include/asm/uaccess.h 2015-10-12 22:33:32.220680454 +0200
@@ -74,7 +74,8 @@
* @addr: User space pointer to start of block to check
* @size: Size of block to check
@@ -3377,7 +3377,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess.h linux-4.1.10/arch/x86
* space. It supports simple types like char and int, but not larger
diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h
--- linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h 2015-10-12 22:33:32.220680454 +0200
@@ -615,9 +615,9 @@
cycles_t send_message;
cycles_t period_end;
@@ -3413,7 +3413,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.10/arch/x
diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h
--- linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h 2015-10-12 22:33:32.220680454 +0200
@@ -492,7 +492,7 @@
unsigned short nr_online_cpus;
unsigned short pnode;
@@ -3425,7 +3425,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.10/arch/x
extern struct uv_blade_info *uv_blade_info;
diff -Nur linux-4.1.10.orig/arch/x86/Kconfig linux-4.1.10/arch/x86/Kconfig
--- linux-4.1.10.orig/arch/x86/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/Kconfig 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/Kconfig 2015-10-12 22:33:32.220680454 +0200
@@ -22,6 +22,7 @@
### Arch settings
config X86
@@ -3458,7 +3458,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/Kconfig linux-4.1.10/arch/x86/Kconfig
If unsure, say N.
diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.10/arch/x86/kernel/apic/io_apic.c
--- linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/apic/io_apic.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/apic/io_apic.c 2015-10-12 22:33:32.220680454 +0200
@@ -1891,7 +1891,8 @@
static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg)
{
@@ -3471,7 +3471,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.10/arch/x86
}
diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c
--- linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-12 22:33:32.220680454 +0200
@@ -949,7 +949,7 @@
uv_blade_info[blade].pnode = pnode;
uv_blade_info[blade].nr_possible_cpus = 0;
@@ -3483,7 +3483,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.10/arch
blade++;
diff -Nur linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c linux-4.1.10/arch/x86/kernel/asm-offsets.c
--- linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/asm-offsets.c 2015-10-12 22:33:32.220680454 +0200
@@ -32,6 +32,7 @@
OFFSET(TI_flags, thread_info, flags);
OFFSET(TI_status, thread_info, status);
@@ -3500,7 +3500,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c linux-4.1.10/arch/x86/
}
diff -Nur linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c
--- linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-12 22:33:32.220680454 +0200
@@ -41,6 +41,8 @@
#include <linux/debugfs.h>
#include <linux/irq_work.h>
@@ -3744,7 +3744,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.10/arch/x
goto err_out;
diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.10/arch/x86/kernel/dumpstack_32.c
--- linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/dumpstack_32.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/dumpstack_32.c 2015-10-12 22:33:32.224680189 +0200
@@ -42,7 +42,7 @@
unsigned long *stack, unsigned long bp,
const struct stacktrace_ops *ops, void *data)
@@ -3765,7 +3765,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.10/arch/x86
diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.10/arch/x86/kernel/dumpstack_64.c
--- linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/dumpstack_64.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/dumpstack_64.c 2015-10-12 22:33:32.224680189 +0200
@@ -152,7 +152,7 @@
unsigned long *stack, unsigned long bp,
const struct stacktrace_ops *ops, void *data)
@@ -3804,7 +3804,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.10/arch/x86
show_trace_log_lvl(task, regs, sp, bp, log_lvl);
diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_32.S linux-4.1.10/arch/x86/kernel/entry_32.S
--- linux-4.1.10.orig/arch/x86/kernel/entry_32.S 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/entry_32.S 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/entry_32.S 2015-10-12 22:33:32.224680189 +0200
@@ -359,8 +359,24 @@
ENTRY(resume_kernel)
DISABLE_INTERRUPTS(CLBR_ANY)
@@ -3850,7 +3850,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_32.S linux-4.1.10/arch/x86/ker
work_notifysig: # deal with pending signals and
diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_64.S linux-4.1.10/arch/x86/kernel/entry_64.S
--- linux-4.1.10.orig/arch/x86/kernel/entry_64.S 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/entry_64.S 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/entry_64.S 2015-10-12 22:33:32.224680189 +0200
@@ -370,8 +370,8 @@
/* First do a reschedule test. */
/* edx: work, edi: workmask */
@@ -3915,7 +3915,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_64.S linux-4.1.10/arch/x86/ker
idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
diff -Nur linux-4.1.10.orig/arch/x86/kernel/irq_32.c linux-4.1.10/arch/x86/kernel/irq_32.c
--- linux-4.1.10.orig/arch/x86/kernel/irq_32.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/irq_32.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/irq_32.c 2015-10-12 22:33:32.224680189 +0200
@@ -135,6 +135,7 @@
cpu, per_cpu(hardirq_stack, cpu), per_cpu(softirq_stack, cpu));
}
@@ -3934,7 +3934,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/irq_32.c linux-4.1.10/arch/x86/kerne
{
diff -Nur linux-4.1.10.orig/arch/x86/kernel/process_32.c linux-4.1.10/arch/x86/kernel/process_32.c
--- linux-4.1.10.orig/arch/x86/kernel/process_32.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/process_32.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/process_32.c 2015-10-12 22:33:32.224680189 +0200
@@ -35,6 +35,7 @@
#include <linux/uaccess.h>
#include <linux/io.h>
@@ -3990,7 +3990,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/process_32.c linux-4.1.10/arch/x86/k
* This must be done before restoring TLS segments so
diff -Nur linux-4.1.10.orig/arch/x86/kernel/signal.c linux-4.1.10/arch/x86/kernel/signal.c
--- linux-4.1.10.orig/arch/x86/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/signal.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/signal.c 2015-10-12 22:33:32.224680189 +0200
@@ -723,6 +723,14 @@
{
user_exit();
@@ -4008,7 +4008,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/signal.c linux-4.1.10/arch/x86/kerne
diff -Nur linux-4.1.10.orig/arch/x86/kernel/traps.c linux-4.1.10/arch/x86/kernel/traps.c
--- linux-4.1.10.orig/arch/x86/kernel/traps.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kernel/traps.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kernel/traps.c 2015-10-12 22:33:32.224680189 +0200
@@ -88,9 +88,21 @@
local_irq_enable();
}
@@ -4085,7 +4085,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/traps.c linux-4.1.10/arch/x86/kernel
exit:
diff -Nur linux-4.1.10.orig/arch/x86/kvm/lapic.c linux-4.1.10/arch/x86/kvm/lapic.c
--- linux-4.1.10.orig/arch/x86/kvm/lapic.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kvm/lapic.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kvm/lapic.c 2015-10-12 22:33:32.224680189 +0200
@@ -1104,7 +1104,7 @@
static void apic_timer_expired(struct kvm_lapic *apic)
{
@@ -4188,7 +4188,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kvm/lapic.c linux-4.1.10/arch/x86/kvm/lapic
/*
diff -Nur linux-4.1.10.orig/arch/x86/kvm/x86.c linux-4.1.10/arch/x86/kvm/x86.c
--- linux-4.1.10.orig/arch/x86/kvm/x86.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/kvm/x86.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/kvm/x86.c 2015-10-12 22:33:32.228679925 +0200
@@ -5813,6 +5813,13 @@
goto out;
}
@@ -4205,7 +4205,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kvm/x86.c linux-4.1.10/arch/x86/kvm/x86.c
goto out_free_percpu;
diff -Nur linux-4.1.10.orig/arch/x86/lib/usercopy_32.c linux-4.1.10/arch/x86/lib/usercopy_32.c
--- linux-4.1.10.orig/arch/x86/lib/usercopy_32.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/lib/usercopy_32.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/lib/usercopy_32.c 2015-10-12 22:33:32.228679925 +0200
@@ -647,7 +647,8 @@
* @from: Source address, in kernel space.
* @n: Number of bytes to copy.
@@ -4228,7 +4228,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/lib/usercopy_32.c linux-4.1.10/arch/x86/lib
*
diff -Nur linux-4.1.10.orig/arch/x86/mm/fault.c linux-4.1.10/arch/x86/mm/fault.c
--- linux-4.1.10.orig/arch/x86/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/mm/fault.c 2015-10-12 22:33:32.228679925 +0200
@@ -13,6 +13,7 @@
#include <linux/hugetlb.h> /* hstate_index_to_shift */
#include <linux/prefetch.h> /* prefetchw */
@@ -4251,7 +4251,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/mm/fault.c linux-4.1.10/arch/x86/mm/fault.c
}
diff -Nur linux-4.1.10.orig/arch/x86/mm/highmem_32.c linux-4.1.10/arch/x86/mm/highmem_32.c
--- linux-4.1.10.orig/arch/x86/mm/highmem_32.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/mm/highmem_32.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/mm/highmem_32.c 2015-10-12 22:33:32.228679925 +0200
@@ -32,10 +32,11 @@
*/
void *kmap_atomic_prot(struct page *page, pgprot_t prot)
@@ -4297,7 +4297,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/mm/highmem_32.c linux-4.1.10/arch/x86/mm/hi
diff -Nur linux-4.1.10.orig/arch/x86/mm/iomap_32.c linux-4.1.10/arch/x86/mm/iomap_32.c
--- linux-4.1.10.orig/arch/x86/mm/iomap_32.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/mm/iomap_32.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/mm/iomap_32.c 2015-10-12 22:33:32.228679925 +0200
@@ -56,15 +56,22 @@
void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot)
@@ -4339,7 +4339,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/mm/iomap_32.c linux-4.1.10/arch/x86/mm/ioma
EXPORT_SYMBOL_GPL(iounmap_atomic);
diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.10/arch/x86/platform/uv/tlb_uv.c
--- linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/platform/uv/tlb_uv.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/platform/uv/tlb_uv.c 2015-10-12 22:33:32.228679925 +0200
@@ -714,9 +714,9 @@
quiesce_local_uvhub(hmaster);
@@ -4428,7 +4428,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.10/arch/x86/
diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c linux-4.1.10/arch/x86/platform/uv/uv_time.c
--- linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/x86/platform/uv/uv_time.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/x86/platform/uv/uv_time.c 2015-10-12 22:33:32.228679925 +0200
@@ -58,7 +58,7 @@
/* There is one of these allocated per node */
@@ -4511,7 +4511,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c linux-4.1.10/arch/x86
/*
diff -Nur linux-4.1.10.orig/arch/xtensa/mm/fault.c linux-4.1.10/arch/xtensa/mm/fault.c
--- linux-4.1.10.orig/arch/xtensa/mm/fault.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/xtensa/mm/fault.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/xtensa/mm/fault.c 2015-10-12 22:33:32.228679925 +0200
@@ -15,10 +15,10 @@
#include <linux/mm.h>
#include <linux/module.h>
@@ -4535,7 +4535,7 @@ diff -Nur linux-4.1.10.orig/arch/xtensa/mm/fault.c linux-4.1.10/arch/xtensa/mm/f
}
diff -Nur linux-4.1.10.orig/arch/xtensa/mm/highmem.c linux-4.1.10/arch/xtensa/mm/highmem.c
--- linux-4.1.10.orig/arch/xtensa/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/arch/xtensa/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/arch/xtensa/mm/highmem.c 2015-10-12 22:33:32.228679925 +0200
@@ -42,6 +42,7 @@
enum fixed_addresses idx;
unsigned long vaddr;
@@ -4554,7 +4554,7 @@ diff -Nur linux-4.1.10.orig/arch/xtensa/mm/highmem.c linux-4.1.10/arch/xtensa/mm
diff -Nur linux-4.1.10.orig/block/blk-core.c linux-4.1.10/block/blk-core.c
--- linux-4.1.10.orig/block/blk-core.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/block/blk-core.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/block/blk-core.c 2015-10-12 22:33:32.228679925 +0200
@@ -100,6 +100,9 @@
INIT_LIST_HEAD(&rq->queuelist);
@@ -4632,7 +4632,7 @@ diff -Nur linux-4.1.10.orig/block/blk-core.c linux-4.1.10/block/blk-core.c
void blk_finish_plug(struct blk_plug *plug)
diff -Nur linux-4.1.10.orig/block/blk-ioc.c linux-4.1.10/block/blk-ioc.c
--- linux-4.1.10.orig/block/blk-ioc.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/block/blk-ioc.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/block/blk-ioc.c 2015-10-12 22:33:32.228679925 +0200
@@ -7,6 +7,7 @@
#include <linux/bio.h>
#include <linux/blkdev.h>
@@ -4661,7 +4661,7 @@ diff -Nur linux-4.1.10.orig/block/blk-ioc.c linux-4.1.10/block/blk-ioc.c
}
diff -Nur linux-4.1.10.orig/block/blk-iopoll.c linux-4.1.10/block/blk-iopoll.c
--- linux-4.1.10.orig/block/blk-iopoll.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/block/blk-iopoll.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/block/blk-iopoll.c 2015-10-12 22:33:32.228679925 +0200
@@ -35,6 +35,7 @@
list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll));
__raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
@@ -4688,7 +4688,7 @@ diff -Nur linux-4.1.10.orig/block/blk-iopoll.c linux-4.1.10/block/blk-iopoll.c
return NOTIFY_OK;
diff -Nur linux-4.1.10.orig/block/blk-mq.c linux-4.1.10/block/blk-mq.c
--- linux-4.1.10.orig/block/blk-mq.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/block/blk-mq.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/block/blk-mq.c 2015-10-12 22:33:32.228679925 +0200
@@ -88,7 +88,7 @@
if (!(gfp & __GFP_WAIT))
return -EBUSY;
@@ -4826,7 +4826,7 @@ diff -Nur linux-4.1.10.orig/block/blk-mq.c linux-4.1.10/block/blk-mq.c
/*
diff -Nur linux-4.1.10.orig/block/blk-mq-cpu.c linux-4.1.10/block/blk-mq-cpu.c
--- linux-4.1.10.orig/block/blk-mq-cpu.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/block/blk-mq-cpu.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/block/blk-mq-cpu.c 2015-10-12 22:33:32.228679925 +0200
@@ -16,7 +16,7 @@
#include "blk-mq.h"
@@ -4880,7 +4880,7 @@ diff -Nur linux-4.1.10.orig/block/blk-mq-cpu.c linux-4.1.10/block/blk-mq-cpu.c
void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier,
diff -Nur linux-4.1.10.orig/block/blk-mq.h linux-4.1.10/block/blk-mq.h
--- linux-4.1.10.orig/block/blk-mq.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/block/blk-mq.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/block/blk-mq.h 2015-10-12 22:33:32.228679925 +0200
@@ -76,7 +76,10 @@
static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
unsigned int cpu)
@@ -4910,7 +4910,7 @@ diff -Nur linux-4.1.10.orig/block/blk-mq.h linux-4.1.10/block/blk-mq.h
struct blk_mq_alloc_data {
diff -Nur linux-4.1.10.orig/block/blk-softirq.c linux-4.1.10/block/blk-softirq.c
--- linux-4.1.10.orig/block/blk-softirq.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/block/blk-softirq.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/block/blk-softirq.c 2015-10-12 22:33:32.228679925 +0200
@@ -51,6 +51,7 @@
raise_softirq_irqoff(BLOCK_SOFTIRQ);
@@ -4937,7 +4937,7 @@ diff -Nur linux-4.1.10.orig/block/blk-softirq.c linux-4.1.10/block/blk-softirq.c
/**
diff -Nur linux-4.1.10.orig/block/bounce.c linux-4.1.10/block/bounce.c
--- linux-4.1.10.orig/block/bounce.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/block/bounce.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/block/bounce.c 2015-10-12 22:33:32.228679925 +0200
@@ -54,11 +54,11 @@
unsigned long flags;
unsigned char *vto;
@@ -4954,7 +4954,7 @@ diff -Nur linux-4.1.10.orig/block/bounce.c linux-4.1.10/block/bounce.c
#else /* CONFIG_HIGHMEM */
diff -Nur linux-4.1.10.orig/crypto/algapi.c linux-4.1.10/crypto/algapi.c
--- linux-4.1.10.orig/crypto/algapi.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/crypto/algapi.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/crypto/algapi.c 2015-10-12 22:33:32.232679661 +0200
@@ -695,13 +695,13 @@
int crypto_register_notifier(struct notifier_block *nb)
@@ -4973,7 +4973,7 @@ diff -Nur linux-4.1.10.orig/crypto/algapi.c linux-4.1.10/crypto/algapi.c
diff -Nur linux-4.1.10.orig/crypto/api.c linux-4.1.10/crypto/api.c
--- linux-4.1.10.orig/crypto/api.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/crypto/api.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/crypto/api.c 2015-10-12 22:33:32.232679661 +0200
@@ -31,7 +31,7 @@
DECLARE_RWSEM(crypto_alg_sem);
EXPORT_SYMBOL_GPL(crypto_alg_sem);
@@ -4998,7 +4998,7 @@ diff -Nur linux-4.1.10.orig/crypto/api.c linux-4.1.10/crypto/api.c
return ok;
diff -Nur linux-4.1.10.orig/crypto/internal.h linux-4.1.10/crypto/internal.h
--- linux-4.1.10.orig/crypto/internal.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/crypto/internal.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/crypto/internal.h 2015-10-12 22:33:32.232679661 +0200
@@ -48,7 +48,7 @@
extern struct list_head crypto_alg_list;
@@ -5019,7 +5019,7 @@ diff -Nur linux-4.1.10.orig/crypto/internal.h linux-4.1.10/crypto/internal.h
#endif /* _CRYPTO_INTERNAL_H */
diff -Nur linux-4.1.10.orig/Documentation/hwlat_detector.txt linux-4.1.10/Documentation/hwlat_detector.txt
--- linux-4.1.10.orig/Documentation/hwlat_detector.txt 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/Documentation/hwlat_detector.txt 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/Documentation/hwlat_detector.txt 2015-10-12 22:33:32.232679661 +0200
@@ -0,0 +1,64 @@
+Introduction:
+-------------
@@ -5087,7 +5087,7 @@ diff -Nur linux-4.1.10.orig/Documentation/hwlat_detector.txt linux-4.1.10/Docume
+consumed by reading from the "sample" (pipe) debugfs file interface.
diff -Nur linux-4.1.10.orig/Documentation/sysrq.txt linux-4.1.10/Documentation/sysrq.txt
--- linux-4.1.10.orig/Documentation/sysrq.txt 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/Documentation/sysrq.txt 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/Documentation/sysrq.txt 2015-10-12 22:33:32.232679661 +0200
@@ -59,10 +59,17 @@
On other - If you know of the key combos for other architectures, please
let me know so I can add them to this section.
@@ -5110,7 +5110,7 @@ diff -Nur linux-4.1.10.orig/Documentation/sysrq.txt linux-4.1.10/Documentation/s
'b' - Will immediately reboot the system without syncing or unmounting
diff -Nur linux-4.1.10.orig/Documentation/trace/histograms.txt linux-4.1.10/Documentation/trace/histograms.txt
--- linux-4.1.10.orig/Documentation/trace/histograms.txt 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/Documentation/trace/histograms.txt 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/Documentation/trace/histograms.txt 2015-10-12 22:33:32.232679661 +0200
@@ -0,0 +1,186 @@
+ Using the Linux Kernel Latency Histograms
+
@@ -5300,7 +5300,7 @@ diff -Nur linux-4.1.10.orig/Documentation/trace/histograms.txt linux-4.1.10/Docu
+These data are also reset when the wakeup histogram is reset.
diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h linux-4.1.10/drivers/acpi/acpica/acglobal.h
--- linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/acpi/acpica/acglobal.h 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/drivers/acpi/acpica/acglobal.h 2015-10-12 22:33:32.232679661 +0200
@@ -112,7 +112,7 @@
* interrupt level
*/
@@ -5312,7 +5312,7 @@ diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h linux-4.1.10/drivers/
/* Mutex for _OSI support */
diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c linux-4.1.10/drivers/acpi/acpica/hwregs.c
--- linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/acpi/acpica/hwregs.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/drivers/acpi/acpica/hwregs.c 2015-10-12 22:33:32.232679661 +0200
@@ -269,14 +269,14 @@
ACPI_BITMASK_ALL_FIXED_STATUS,
ACPI_FORMAT_UINT64(acpi_gbl_xpm1a_status.address)));
@@ -5332,7 +5332,7 @@ diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c linux-4.1.10/drivers/ac
goto exit;
diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c linux-4.1.10/drivers/acpi/acpica/hwxface.c
--- linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/acpi/acpica/hwxface.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/drivers/acpi/acpica/hwxface.c 2015-10-12 22:33:32.232679661 +0200
@@ -374,7 +374,7 @@
return_ACPI_STATUS(AE_BAD_PARAMETER);
}
@@ -5353,7 +5353,7 @@ diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c linux-4.1.10/drivers/a
diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c linux-4.1.10/drivers/acpi/acpica/utmutex.c
--- linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/acpi/acpica/utmutex.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/drivers/acpi/acpica/utmutex.c 2015-10-12 22:33:32.232679661 +0200
@@ -88,7 +88,7 @@
return_ACPI_STATUS (status);
}
@@ -5374,7 +5374,7 @@ diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c linux-4.1.10/drivers/a
/* Delete the reader/writer lock */
diff -Nur linux-4.1.10.orig/drivers/ata/libata-sff.c linux-4.1.10/drivers/ata/libata-sff.c
--- linux-4.1.10.orig/drivers/ata/libata-sff.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/ata/libata-sff.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/drivers/ata/libata-sff.c 2015-10-12 22:33:32.232679661 +0200
@@ -678,9 +678,9 @@
unsigned long flags;
unsigned int consumed;
@@ -5425,7 +5425,7 @@ diff -Nur linux-4.1.10.orig/drivers/ata/libata-sff.c linux-4.1.10/drivers/ata/li
consumed = ap->ops->sff_data_xfer(dev, buf + offset,
diff -Nur linux-4.1.10.orig/drivers/char/random.c linux-4.1.10/drivers/char/random.c
--- linux-4.1.10.orig/drivers/char/random.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/char/random.c 2015-10-07 18:00:07.000000000 +0200
++++ linux-4.1.10/drivers/char/random.c 2015-10-12 22:33:32.232679661 +0200
@@ -776,8 +776,6 @@
} sample;
long delta, delta2, delta3;
@@ -5479,7 +5479,7 @@ diff -Nur linux-4.1.10.orig/drivers/char/random.c linux-4.1.10/drivers/char/rand
add_interrupt_bench(cycles);
diff -Nur linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.10/drivers/clocksource/tcb_clksrc.c
--- linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/clocksource/tcb_clksrc.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/clocksource/tcb_clksrc.c 2015-10-12 22:33:32.232679661 +0200
@@ -23,8 +23,7 @@
* this 32 bit free-running counter. the second channel is not used.
*
@@ -5596,7 +5596,7 @@ diff -Nur linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.10/driver
diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.10/drivers/clocksource/timer-atmel-pit.c
--- linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/clocksource/timer-atmel-pit.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/clocksource/timer-atmel-pit.c 2015-10-12 22:33:32.232679661 +0200
@@ -90,6 +90,7 @@
return elapsed;
}
@@ -5624,7 +5624,7 @@ diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.10/d
break;
diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.10/drivers/clocksource/timer-atmel-st.c
--- linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/clocksource/timer-atmel-st.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/clocksource/timer-atmel-st.c 2015-10-12 22:33:32.232679661 +0200
@@ -131,6 +131,7 @@
break;
case CLOCK_EVT_MODE_SHUTDOWN:
@@ -5635,7 +5635,7 @@ diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.10/dr
break;
diff -Nur linux-4.1.10.orig/drivers/cpufreq/cpufreq.c linux-4.1.10/drivers/cpufreq/cpufreq.c
--- linux-4.1.10.orig/drivers/cpufreq/cpufreq.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/cpufreq/cpufreq.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/cpufreq/cpufreq.c 2015-10-12 22:33:32.232679661 +0200
@@ -64,12 +64,6 @@
return cpufreq_driver->target_index || cpufreq_driver->target;
}
@@ -5774,7 +5774,7 @@ diff -Nur linux-4.1.10.orig/drivers/cpufreq/cpufreq.c linux-4.1.10/drivers/cpufr
}
diff -Nur linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.10/drivers/cpufreq/Kconfig.x86
--- linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/cpufreq/Kconfig.x86 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/cpufreq/Kconfig.x86 2015-10-12 22:33:32.232679661 +0200
@@ -123,7 +123,7 @@
config X86_POWERNOW_K8
@@ -5786,7 +5786,7 @@ diff -Nur linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.10/drivers/cpu
Support for K10 and newer processors is now in acpi-cpufreq.
diff -Nur linux-4.1.10.orig/drivers/gpio/gpio-omap.c linux-4.1.10/drivers/gpio/gpio-omap.c
--- linux-4.1.10.orig/drivers/gpio/gpio-omap.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/gpio/gpio-omap.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/gpio/gpio-omap.c 2015-10-12 22:33:32.236679397 +0200
@@ -57,7 +57,7 @@
u32 saved_datain;
u32 level_mask;
@@ -6069,7 +6069,7 @@ diff -Nur linux-4.1.10.orig/drivers/gpio/gpio-omap.c linux-4.1.10/drivers/gpio/g
}
diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c
--- linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-12 22:33:32.236679397 +0200
@@ -32,6 +32,7 @@
#include "i915_trace.h"
#include "intel_drv.h"
@@ -6099,7 +6099,7 @@ diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1
i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c
--- linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-12 22:33:32.236679397 +0200
@@ -39,7 +39,7 @@
if (!mutex_is_locked(mutex))
return false;
@@ -6111,7 +6111,7 @@ diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.1
/* Since UP may be pre-empted, we cannot assume that we own the lock */
diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.10/drivers/gpu/drm/i915/intel_display.c
--- linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/gpu/drm/i915/intel_display.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/gpu/drm/i915/intel_display.c 2015-10-12 22:33:32.236679397 +0200
@@ -10086,7 +10086,7 @@
struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
@@ -6123,7 +6123,7 @@ diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.10/dr
return;
diff -Nur linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.10/drivers/i2c/busses/i2c-omap.c
--- linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/i2c/busses/i2c-omap.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/i2c/busses/i2c-omap.c 2015-10-12 22:33:32.244678868 +0200
@@ -996,15 +996,12 @@
u16 mask;
u16 stat;
@@ -6143,7 +6143,7 @@ diff -Nur linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.10/drivers/i
diff -Nur linux-4.1.10.orig/drivers/ide/alim15x3.c linux-4.1.10/drivers/ide/alim15x3.c
--- linux-4.1.10.orig/drivers/ide/alim15x3.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/ide/alim15x3.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/ide/alim15x3.c 2015-10-12 22:33:32.244678868 +0200
@@ -234,7 +234,7 @@
isa_dev = pci_get_device(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, NULL);
@@ -6164,7 +6164,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/alim15x3.c linux-4.1.10/drivers/ide/alim
diff -Nur linux-4.1.10.orig/drivers/ide/hpt366.c linux-4.1.10/drivers/ide/hpt366.c
--- linux-4.1.10.orig/drivers/ide/hpt366.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/ide/hpt366.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/ide/hpt366.c 2015-10-12 22:33:32.244678868 +0200
@@ -1241,7 +1241,7 @@
dma_old = inb(base + 2);
@@ -6185,7 +6185,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/hpt366.c linux-4.1.10/drivers/ide/hpt366
hwif->name, base, base + 7);
diff -Nur linux-4.1.10.orig/drivers/ide/ide-io.c linux-4.1.10/drivers/ide/ide-io.c
--- linux-4.1.10.orig/drivers/ide/ide-io.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/ide/ide-io.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/ide/ide-io.c 2015-10-12 22:33:32.244678868 +0200
@@ -659,7 +659,7 @@
/* disable_irq_nosync ?? */
disable_irq(hwif->irq);
@@ -6197,7 +6197,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-io.c linux-4.1.10/drivers/ide/ide-io
} else if (drive_is_ready(drive)) {
diff -Nur linux-4.1.10.orig/drivers/ide/ide-iops.c linux-4.1.10/drivers/ide/ide-iops.c
--- linux-4.1.10.orig/drivers/ide/ide-iops.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/ide/ide-iops.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/ide/ide-iops.c 2015-10-12 22:33:32.244678868 +0200
@@ -129,12 +129,12 @@
if ((stat & ATA_BUSY) == 0)
break;
@@ -6215,7 +6215,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-iops.c linux-4.1.10/drivers/ide/ide-
* Allow status to settle, then read it again.
diff -Nur linux-4.1.10.orig/drivers/ide/ide-io-std.c linux-4.1.10/drivers/ide/ide-io-std.c
--- linux-4.1.10.orig/drivers/ide/ide-io-std.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/ide/ide-io-std.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/ide/ide-io-std.c 2015-10-12 22:33:32.244678868 +0200
@@ -175,7 +175,7 @@
unsigned long uninitialized_var(flags);
@@ -6254,7 +6254,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-io-std.c linux-4.1.10/drivers/ide/id
return;
diff -Nur linux-4.1.10.orig/drivers/ide/ide-probe.c linux-4.1.10/drivers/ide/ide-probe.c
--- linux-4.1.10.orig/drivers/ide/ide-probe.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/ide/ide-probe.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/ide/ide-probe.c 2015-10-12 22:33:32.244678868 +0200
@@ -196,10 +196,10 @@
int bswap = 1;
@@ -6270,7 +6270,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-probe.c linux-4.1.10/drivers/ide/ide
#ifdef DEBUG
diff -Nur linux-4.1.10.orig/drivers/ide/ide-taskfile.c linux-4.1.10/drivers/ide/ide-taskfile.c
--- linux-4.1.10.orig/drivers/ide/ide-taskfile.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/ide/ide-taskfile.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/ide/ide-taskfile.c 2015-10-12 22:33:32.244678868 +0200
@@ -250,7 +250,7 @@
page_is_high = PageHighMem(page);
@@ -6300,7 +6300,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-taskfile.c linux-4.1.10/drivers/ide/
diff -Nur linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
--- linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-12 22:33:32.244678868 +0200
@@ -821,7 +821,7 @@
ipoib_dbg_mcast(priv, "restarting multicast task\n");
@@ -6321,7 +6321,7 @@ diff -Nur linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux
* make sure the in-flight joins have finished before we attempt
diff -Nur linux-4.1.10.orig/drivers/input/gameport/gameport.c linux-4.1.10/drivers/input/gameport/gameport.c
--- linux-4.1.10.orig/drivers/input/gameport/gameport.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/input/gameport/gameport.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/input/gameport/gameport.c 2015-10-12 22:33:32.244678868 +0200
@@ -124,12 +124,12 @@
tx = 1 << 30;
@@ -6353,7 +6353,7 @@ diff -Nur linux-4.1.10.orig/drivers/input/gameport/gameport.c linux-4.1.10/drive
}
diff -Nur linux-4.1.10.orig/drivers/leds/trigger/Kconfig linux-4.1.10/drivers/leds/trigger/Kconfig
--- linux-4.1.10.orig/drivers/leds/trigger/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/leds/trigger/Kconfig 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/leds/trigger/Kconfig 2015-10-12 22:33:32.244678868 +0200
@@ -61,7 +61,7 @@
config LEDS_TRIGGER_CPU
@@ -6365,7 +6365,7 @@ diff -Nur linux-4.1.10.orig/drivers/leds/trigger/Kconfig linux-4.1.10/drivers/le
the active CPUs across an array of LEDs so you can see which
diff -Nur linux-4.1.10.orig/drivers/md/bcache/Kconfig linux-4.1.10/drivers/md/bcache/Kconfig
--- linux-4.1.10.orig/drivers/md/bcache/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/md/bcache/Kconfig 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/md/bcache/Kconfig 2015-10-12 22:33:32.244678868 +0200
@@ -1,6 +1,7 @@
config BCACHE
@@ -6376,7 +6376,7 @@ diff -Nur linux-4.1.10.orig/drivers/md/bcache/Kconfig linux-4.1.10/drivers/md/bc
a btree for indexing and the layout is optimized for SSDs.
diff -Nur linux-4.1.10.orig/drivers/md/dm.c linux-4.1.10/drivers/md/dm.c
--- linux-4.1.10.orig/drivers/md/dm.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/md/dm.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/md/dm.c 2015-10-12 22:33:32.244678868 +0200
@@ -2132,7 +2132,7 @@
/* Establish tio->ti before queuing work (map_tio_request) */
tio->ti = ti;
@@ -6388,7 +6388,7 @@ diff -Nur linux-4.1.10.orig/drivers/md/dm.c linux-4.1.10/drivers/md/dm.c
goto out;
diff -Nur linux-4.1.10.orig/drivers/md/raid5.c linux-4.1.10/drivers/md/raid5.c
--- linux-4.1.10.orig/drivers/md/raid5.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/md/raid5.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/md/raid5.c 2015-10-12 22:33:32.248678604 +0200
@@ -1918,8 +1918,9 @@
struct raid5_percpu *percpu;
unsigned long cpu;
@@ -6418,7866 +6418,9 @@ diff -Nur linux-4.1.10.orig/drivers/md/raid5.c linux-4.1.10/drivers/md/raid5.c
}
put_online_cpus();
-diff -Nur linux-4.1.10.orig/drivers/md/raid5.c.orig linux-4.1.10/drivers/md/raid5.c.orig
---- linux-4.1.10.orig/drivers/md/raid5.c.orig 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/drivers/md/raid5.c.orig 2015-10-03 13:49:38.000000000 +0200
-@@ -0,0 +1,7853 @@
-+/*
-+ * raid5.c : Multiple Devices driver for Linux
-+ * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
-+ * Copyright (C) 1999, 2000 Ingo Molnar
-+ * Copyright (C) 2002, 2003 H. Peter Anvin
-+ *
-+ * RAID-4/5/6 management functions.
-+ * Thanks to Penguin Computing for making the RAID-6 development possible
-+ * by donating a test server!
-+ *
-+ * This program is free software; you can redistribute it and/or modify
-+ * it under the terms of the GNU General Public License as published by
-+ * the Free Software Foundation; either version 2, or (at your option)
-+ * any later version.
-+ *
-+ * You should have received a copy of the GNU General Public License
-+ * (for example /usr/src/linux/COPYING); if not, write to the Free
-+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-+ */
-+
-+/*
-+ * BITMAP UNPLUGGING:
-+ *
-+ * The sequencing for updating the bitmap reliably is a little
-+ * subtle (and I got it wrong the first time) so it deserves some
-+ * explanation.
-+ *
-+ * We group bitmap updates into batches. Each batch has a number.
-+ * We may write out several batches at once, but that isn't very important.
-+ * conf->seq_write is the number of the last batch successfully written.
-+ * conf->seq_flush is the number of the last batch that was closed to
-+ * new additions.
-+ * When we discover that we will need to write to any block in a stripe
-+ * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
-+ * the number of the batch it will be in. This is seq_flush+1.
-+ * When we are ready to do a write, if that batch hasn't been written yet,
-+ * we plug the array and queue the stripe for later.
-+ * When an unplug happens, we increment bm_flush, thus closing the current
-+ * batch.
-+ * When we notice that bm_flush > bm_write, we write out all pending updates
-+ * to the bitmap, and advance bm_write to where bm_flush was.
-+ * This may occasionally write a bit out twice, but is sure never to
-+ * miss any bits.
-+ */
-+
-+#include <linux/blkdev.h>
-+#include <linux/kthread.h>
-+#include <linux/raid/pq.h>
-+#include <linux/async_tx.h>
-+#include <linux/module.h>
-+#include <linux/async.h>
-+#include <linux/seq_file.h>
-+#include <linux/cpu.h>
-+#include <linux/slab.h>
-+#include <linux/ratelimit.h>
-+#include <linux/nodemask.h>
-+#include <linux/flex_array.h>
-+#include <trace/events/block.h>
-+
-+#include "md.h"
-+#include "raid5.h"
-+#include "raid0.h"
-+#include "bitmap.h"
-+
-+#define cpu_to_group(cpu) cpu_to_node(cpu)
-+#define ANY_GROUP NUMA_NO_NODE
-+
-+static bool devices_handle_discard_safely = false;
-+module_param(devices_handle_discard_safely, bool, 0644);
-+MODULE_PARM_DESC(devices_handle_discard_safely,
-+ "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
-+static struct workqueue_struct *raid5_wq;
-+/*
-+ * Stripe cache
-+ */
-+
-+#define NR_STRIPES 256
-+#define STRIPE_SIZE PAGE_SIZE
-+#define STRIPE_SHIFT (PAGE_SHIFT - 9)
-+#define STRIPE_SECTORS (STRIPE_SIZE>>9)
-+#define IO_THRESHOLD 1
-+#define BYPASS_THRESHOLD 1
-+#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
-+#define HASH_MASK (NR_HASH - 1)
-+#define MAX_STRIPE_BATCH 8
-+
-+static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
-+{
-+ int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
-+ return &conf->stripe_hashtbl[hash];
-+}
-+
-+static inline int stripe_hash_locks_hash(sector_t sect)
-+{
-+ return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
-+}
-+
-+static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
-+{
-+ spin_lock_irq(conf->hash_locks + hash);
-+ spin_lock(&conf->device_lock);
-+}
-+
-+static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
-+{
-+ spin_unlock(&conf->device_lock);
-+ spin_unlock_irq(conf->hash_locks + hash);
-+}
-+
-+static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
-+{
-+ int i;
-+ local_irq_disable();
-+ spin_lock(conf->hash_locks);
-+ for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
-+ spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
-+ spin_lock(&conf->device_lock);
-+}
-+
-+static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
-+{
-+ int i;
-+ spin_unlock(&conf->device_lock);
-+ for (i = NR_STRIPE_HASH_LOCKS; i; i--)
-+ spin_unlock(conf->hash_locks + i - 1);
-+ local_irq_enable();
-+}
-+
-+/* bio's attached to a stripe+device for I/O are linked together in bi_sector
-+ * order without overlap. There may be several bio's per stripe+device, and
-+ * a bio could span several devices.
-+ * When walking this list for a particular stripe+device, we must never proceed
-+ * beyond a bio that extends past this device, as the next bio might no longer
-+ * be valid.
-+ * This function is used to determine the 'next' bio in the list, given the sector
-+ * of the current stripe+device
-+ */
-+static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
-+{
-+ int sectors = bio_sectors(bio);
-+ if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
-+ return bio->bi_next;
-+ else
-+ return NULL;
-+}
-+
-+/*
-+ * We maintain a biased count of active stripes in the bottom 16 bits of
-+ * bi_phys_segments, and a count of processed stripes in the upper 16 bits
-+ */
-+static inline int raid5_bi_processed_stripes(struct bio *bio)
-+{
-+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-+ return (atomic_read(segments) >> 16) & 0xffff;
-+}
-+
-+static inline int raid5_dec_bi_active_stripes(struct bio *bio)
-+{
-+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-+ return atomic_sub_return(1, segments) & 0xffff;
-+}
-+
-+static inline void raid5_inc_bi_active_stripes(struct bio *bio)
-+{
-+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-+ atomic_inc(segments);
-+}
-+
-+static inline void raid5_set_bi_processed_stripes(struct bio *bio,
-+ unsigned int cnt)
-+{
-+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-+ int old, new;
-+
-+ do {
-+ old = atomic_read(segments);
-+ new = (old & 0xffff) | (cnt << 16);
-+ } while (atomic_cmpxchg(segments, old, new) != old);
-+}
-+
-+static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
-+{
-+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-+ atomic_set(segments, cnt);
-+}
-+
-+/* Find first data disk in a raid6 stripe */
-+static inline int raid6_d0(struct stripe_head *sh)
-+{
-+ if (sh->ddf_layout)
-+ /* ddf always start from first device */
-+ return 0;
-+ /* md starts just after Q block */
-+ if (sh->qd_idx == sh->disks - 1)
-+ return 0;
-+ else
-+ return sh->qd_idx + 1;
-+}
-+static inline int raid6_next_disk(int disk, int raid_disks)
-+{
-+ disk++;
-+ return (disk < raid_disks) ? disk : 0;
-+}
-+
-+/* When walking through the disks in a raid5, starting at raid6_d0,
-+ * We need to map each disk to a 'slot', where the data disks are slot
-+ * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
-+ * is raid_disks-1. This help does that mapping.
-+ */
-+static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
-+ int *count, int syndrome_disks)
-+{
-+ int slot = *count;
-+
-+ if (sh->ddf_layout)
-+ (*count)++;
-+ if (idx == sh->pd_idx)
-+ return syndrome_disks;
-+ if (idx == sh->qd_idx)
-+ return syndrome_disks + 1;
-+ if (!sh->ddf_layout)
-+ (*count)++;
-+ return slot;
-+}
-+
-+static void return_io(struct bio *return_bi)
-+{
-+ struct bio *bi = return_bi;
-+ while (bi) {
-+
-+ return_bi = bi->bi_next;
-+ bi->bi_next = NULL;
-+ bi->bi_iter.bi_size = 0;
-+ trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
-+ bi, 0);
-+ bio_endio(bi, 0);
-+ bi = return_bi;
-+ }
-+}
-+
-+static void print_raid5_conf (struct r5conf *conf);
-+
-+static int stripe_operations_active(struct stripe_head *sh)
-+{
-+ return sh->check_state || sh->reconstruct_state ||
-+ test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
-+ test_bit(STRIPE_COMPUTE_RUN, &sh->state);
-+}
-+
-+static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
-+{
-+ struct r5conf *conf = sh->raid_conf;
-+ struct r5worker_group *group;
-+ int thread_cnt;
-+ int i, cpu = sh->cpu;
-+
-+ if (!cpu_online(cpu)) {
-+ cpu = cpumask_any(cpu_online_mask);
-+ sh->cpu = cpu;
-+ }
-+
-+ if (list_empty(&sh->lru)) {
-+ struct r5worker_group *group;
-+ group = conf->worker_groups + cpu_to_group(cpu);
-+ list_add_tail(&sh->lru, &group->handle_list);
-+ group->stripes_cnt++;
-+ sh->group = group;
-+ }
-+
-+ if (conf->worker_cnt_per_group == 0) {
-+ md_wakeup_thread(conf->mddev->thread);
-+ return;
-+ }
-+
-+ group = conf->worker_groups + cpu_to_group(sh->cpu);
-+
-+ group->workers[0].working = true;
-+ /* at least one worker should run to avoid race */
-+ queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
-+
-+ thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
-+ /* wakeup more workers */
-+ for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
-+ if (group->workers[i].working == false) {
-+ group->workers[i].working = true;
-+ queue_work_on(sh->cpu, raid5_wq,
-+ &group->workers[i].work);
-+ thread_cnt--;
-+ }
-+ }
-+}
-+
-+static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
-+ struct list_head *temp_inactive_list)
-+{
-+ BUG_ON(!list_empty(&sh->lru));
-+ BUG_ON(atomic_read(&conf->active_stripes)==0);
-+ if (test_bit(STRIPE_HANDLE, &sh->state)) {
-+ if (test_bit(STRIPE_DELAYED, &sh->state) &&
-+ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-+ list_add_tail(&sh->lru, &conf->delayed_list);
-+ else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
-+ sh->bm_seq - conf->seq_write > 0)
-+ list_add_tail(&sh->lru, &conf->bitmap_list);
-+ else {
-+ clear_bit(STRIPE_DELAYED, &sh->state);
-+ clear_bit(STRIPE_BIT_DELAY, &sh->state);
-+ if (conf->worker_cnt_per_group == 0) {
-+ list_add_tail(&sh->lru, &conf->handle_list);
-+ } else {
-+ raid5_wakeup_stripe_thread(sh);
-+ return;
-+ }
-+ }
-+ md_wakeup_thread(conf->mddev->thread);
-+ } else {
-+ BUG_ON(stripe_operations_active(sh));
-+ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-+ if (atomic_dec_return(&conf->preread_active_stripes)
-+ < IO_THRESHOLD)
-+ md_wakeup_thread(conf->mddev->thread);
-+ atomic_dec(&conf->active_stripes);
-+ if (!test_bit(STRIPE_EXPANDING, &sh->state))
-+ list_add_tail(&sh->lru, temp_inactive_list);
-+ }
-+}
-+
-+static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
-+ struct list_head *temp_inactive_list)
-+{
-+ if (atomic_dec_and_test(&sh->count))
-+ do_release_stripe(conf, sh, temp_inactive_list);
-+}
-+
-+/*
-+ * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
-+ *
-+ * Be careful: Only one task can add/delete stripes from temp_inactive_list at
-+ * given time. Adding stripes only takes device lock, while deleting stripes
-+ * only takes hash lock.
-+ */
-+static void release_inactive_stripe_list(struct r5conf *conf,
-+ struct list_head *temp_inactive_list,
-+ int hash)
-+{
-+ int size;
-+ bool do_wakeup = false;
-+ unsigned long flags;
-+
-+ if (hash == NR_STRIPE_HASH_LOCKS) {
-+ size = NR_STRIPE_HASH_LOCKS;
-+ hash = NR_STRIPE_HASH_LOCKS - 1;
-+ } else
-+ size = 1;
-+ while (size) {
-+ struct list_head *list = &temp_inactive_list[size - 1];
-+
-+ /*
-+ * We don't hold any lock here yet, get_active_stripe() might
-+ * remove stripes from the list
-+ */
-+ if (!list_empty_careful(list)) {
-+ spin_lock_irqsave(conf->hash_locks + hash, flags);
-+ if (list_empty(conf->inactive_list + hash) &&
-+ !list_empty(list))
-+ atomic_dec(&conf->empty_inactive_list_nr);
-+ list_splice_tail_init(list, conf->inactive_list + hash);
-+ do_wakeup = true;
-+ spin_unlock_irqrestore(conf->hash_locks + hash, flags);
-+ }
-+ size--;
-+ hash--;
-+ }
-+
-+ if (do_wakeup) {
-+ wake_up(&conf->wait_for_stripe);
-+ if (conf->retry_read_aligned)
-+ md_wakeup_thread(conf->mddev->thread);
-+ }
-+}
-+
-+/* should hold conf->device_lock already */
-+static int release_stripe_list(struct r5conf *conf,
-+ struct list_head *temp_inactive_list)
-+{
-+ struct stripe_head *sh;
-+ int count = 0;
-+ struct llist_node *head;
-+
-+ head = llist_del_all(&conf->released_stripes);
-+ head = llist_reverse_order(head);
-+ while (head) {
-+ int hash;
-+
-+ sh = llist_entry(head, struct stripe_head, release_list);
-+ head = llist_next(head);
-+ /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
-+ smp_mb();
-+ clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
-+ /*
-+ * Don't worry the bit is set here, because if the bit is set
-+ * again, the count is always > 1. This is true for
-+ * STRIPE_ON_UNPLUG_LIST bit too.
-+ */
-+ hash = sh->hash_lock_index;
-+ __release_stripe(conf, sh, &temp_inactive_list[hash]);
-+ count++;
-+ }
-+
-+ return count;
-+}
-+
-+static void release_stripe(struct stripe_head *sh)
-+{
-+ struct r5conf *conf = sh->raid_conf;
-+ unsigned long flags;
-+ struct list_head list;
-+ int hash;
-+ bool wakeup;
-+
-+ /* Avoid release_list until the last reference.
-+ */
-+ if (atomic_add_unless(&sh->count, -1, 1))
-+ return;
-+
-+ if (unlikely(!conf->mddev->thread) ||
-+ test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
-+ goto slow_path;
-+ wakeup = llist_add(&sh->release_list, &conf->released_stripes);
-+ if (wakeup)
-+ md_wakeup_thread(conf->mddev->thread);
-+ return;
-+slow_path:
-+ local_irq_save(flags);
-+ /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
-+ if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
-+ INIT_LIST_HEAD(&list);
-+ hash = sh->hash_lock_index;
-+ do_release_stripe(conf, sh, &list);
-+ spin_unlock(&conf->device_lock);
-+ release_inactive_stripe_list(conf, &list, hash);
-+ }
-+ local_irq_restore(flags);
-+}
-+
-+static inline void remove_hash(struct stripe_head *sh)
-+{
-+ pr_debug("remove_hash(), stripe %llu\n",
-+ (unsigned long long)sh->sector);
-+
-+ hlist_del_init(&sh->hash);
-+}
-+
-+static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
-+{
-+ struct hlist_head *hp = stripe_hash(conf, sh->sector);
-+
-+ pr_debug("insert_hash(), stripe %llu\n",
-+ (unsigned long long)sh->sector);
-+
-+ hlist_add_head(&sh->hash, hp);
-+}
-+
-+/* find an idle stripe, make sure it is unhashed, and return it. */
-+static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
-+{
-+ struct stripe_head *sh = NULL;
-+ struct list_head *first;
-+
-+ if (list_empty(conf->inactive_list + hash))
-+ goto out;
-+ first = (conf->inactive_list + hash)->next;
-+ sh = list_entry(first, struct stripe_head, lru);
-+ list_del_init(first);
-+ remove_hash(sh);
-+ atomic_inc(&conf->active_stripes);
-+ BUG_ON(hash != sh->hash_lock_index);
-+ if (list_empty(conf->inactive_list + hash))
-+ atomic_inc(&conf->empty_inactive_list_nr);
-+out:
-+ return sh;
-+}
-+
-+static void shrink_buffers(struct stripe_head *sh)
-+{
-+ struct page *p;
-+ int i;
-+ int num = sh->raid_conf->pool_size;
-+
-+ for (i = 0; i < num ; i++) {
-+ WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
-+ p = sh->dev[i].page;
-+ if (!p)
-+ continue;
-+ sh->dev[i].page = NULL;
-+ put_page(p);
-+ }
-+}
-+
-+static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
-+{
-+ int i;
-+ int num = sh->raid_conf->pool_size;
-+
-+ for (i = 0; i < num; i++) {
-+ struct page *page;
-+
-+ if (!(page = alloc_page(gfp))) {
-+ return 1;
-+ }
-+ sh->dev[i].page = page;
-+ sh->dev[i].orig_page = page;
-+ }
-+ return 0;
-+}
-+
-+static void raid5_build_block(struct stripe_head *sh, int i, int previous);
-+static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
-+ struct stripe_head *sh);
-+
-+static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
-+{
-+ struct r5conf *conf = sh->raid_conf;
-+ int i, seq;
-+
-+ BUG_ON(atomic_read(&sh->count) != 0);
-+ BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
-+ BUG_ON(stripe_operations_active(sh));
-+ BUG_ON(sh->batch_head);
-+
-+ pr_debug("init_stripe called, stripe %llu\n",
-+ (unsigned long long)sector);
-+retry:
-+ seq = read_seqcount_begin(&conf->gen_lock);
-+ sh->generation = conf->generation - previous;
-+ sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
-+ sh->sector = sector;
-+ stripe_set_idx(sector, conf, previous, sh);
-+ sh->state = 0;
-+
-+ for (i = sh->disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+
-+ if (dev->toread || dev->read || dev->towrite || dev->written ||
-+ test_bit(R5_LOCKED, &dev->flags)) {
-+ printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
-+ (unsigned long long)sh->sector, i, dev->toread,
-+ dev->read, dev->towrite, dev->written,
-+ test_bit(R5_LOCKED, &dev->flags));
-+ WARN_ON(1);
-+ }
-+ dev->flags = 0;
-+ raid5_build_block(sh, i, previous);
-+ }
-+ if (read_seqcount_retry(&conf->gen_lock, seq))
-+ goto retry;
-+ sh->overwrite_disks = 0;
-+ insert_hash(conf, sh);
-+ sh->cpu = smp_processor_id();
-+ set_bit(STRIPE_BATCH_READY, &sh->state);
-+}
-+
-+static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
-+ short generation)
-+{
-+ struct stripe_head *sh;
-+
-+ pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
-+ hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
-+ if (sh->sector == sector && sh->generation == generation)
-+ return sh;
-+ pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
-+ return NULL;
-+}
-+
-+/*
-+ * Need to check if array has failed when deciding whether to:
-+ * - start an array
-+ * - remove non-faulty devices
-+ * - add a spare
-+ * - allow a reshape
-+ * This determination is simple when no reshape is happening.
-+ * However if there is a reshape, we need to carefully check
-+ * both the before and after sections.
-+ * This is because some failed devices may only affect one
-+ * of the two sections, and some non-in_sync devices may
-+ * be insync in the section most affected by failed devices.
-+ */
-+static int calc_degraded(struct r5conf *conf)
-+{
-+ int degraded, degraded2;
-+ int i;
-+
-+ rcu_read_lock();
-+ degraded = 0;
-+ for (i = 0; i < conf->previous_raid_disks; i++) {
-+ struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
-+ if (rdev && test_bit(Faulty, &rdev->flags))
-+ rdev = rcu_dereference(conf->disks[i].replacement);
-+ if (!rdev || test_bit(Faulty, &rdev->flags))
-+ degraded++;
-+ else if (test_bit(In_sync, &rdev->flags))
-+ ;
-+ else
-+ /* not in-sync or faulty.
-+ * If the reshape increases the number of devices,
-+ * this is being recovered by the reshape, so
-+ * this 'previous' section is not in_sync.
-+ * If the number of devices is being reduced however,
-+ * the device can only be part of the array if
-+ * we are reverting a reshape, so this section will
-+ * be in-sync.
-+ */
-+ if (conf->raid_disks >= conf->previous_raid_disks)
-+ degraded++;
-+ }
-+ rcu_read_unlock();
-+ if (conf->raid_disks == conf->previous_raid_disks)
-+ return degraded;
-+ rcu_read_lock();
-+ degraded2 = 0;
-+ for (i = 0; i < conf->raid_disks; i++) {
-+ struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
-+ if (rdev && test_bit(Faulty, &rdev->flags))
-+ rdev = rcu_dereference(conf->disks[i].replacement);
-+ if (!rdev || test_bit(Faulty, &rdev->flags))
-+ degraded2++;
-+ else if (test_bit(In_sync, &rdev->flags))
-+ ;
-+ else
-+ /* not in-sync or faulty.
-+ * If reshape increases the number of devices, this
-+ * section has already been recovered, else it
-+ * almost certainly hasn't.
-+ */
-+ if (conf->raid_disks <= conf->previous_raid_disks)
-+ degraded2++;
-+ }
-+ rcu_read_unlock();
-+ if (degraded2 > degraded)
-+ return degraded2;
-+ return degraded;
-+}
-+
-+static int has_failed(struct r5conf *conf)
-+{
-+ int degraded;
-+
-+ if (conf->mddev->reshape_position == MaxSector)
-+ return conf->mddev->degraded > conf->max_degraded;
-+
-+ degraded = calc_degraded(conf);
-+ if (degraded > conf->max_degraded)
-+ return 1;
-+ return 0;
-+}
-+
-+static struct stripe_head *
-+get_active_stripe(struct r5conf *conf, sector_t sector,
-+ int previous, int noblock, int noquiesce)
-+{
-+ struct stripe_head *sh;
-+ int hash = stripe_hash_locks_hash(sector);
-+
-+ pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
-+
-+ spin_lock_irq(conf->hash_locks + hash);
-+
-+ do {
-+ wait_event_lock_irq(conf->wait_for_stripe,
-+ conf->quiesce == 0 || noquiesce,
-+ *(conf->hash_locks + hash));
-+ sh = __find_stripe(conf, sector, conf->generation - previous);
-+ if (!sh) {
-+ if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {
-+ sh = get_free_stripe(conf, hash);
-+ if (!sh && llist_empty(&conf->released_stripes) &&
-+ !test_bit(R5_DID_ALLOC, &conf->cache_state))
-+ set_bit(R5_ALLOC_MORE,
-+ &conf->cache_state);
-+ }
-+ if (noblock && sh == NULL)
-+ break;
-+ if (!sh) {
-+ set_bit(R5_INACTIVE_BLOCKED,
-+ &conf->cache_state);
-+ wait_event_lock_irq(
-+ conf->wait_for_stripe,
-+ !list_empty(conf->inactive_list + hash) &&
-+ (atomic_read(&conf->active_stripes)
-+ < (conf->max_nr_stripes * 3 / 4)
-+ || !test_bit(R5_INACTIVE_BLOCKED,
-+ &conf->cache_state)),
-+ *(conf->hash_locks + hash));
-+ clear_bit(R5_INACTIVE_BLOCKED,
-+ &conf->cache_state);
-+ } else {
-+ init_stripe(sh, sector, previous);
-+ atomic_inc(&sh->count);
-+ }
-+ } else if (!atomic_inc_not_zero(&sh->count)) {
-+ spin_lock(&conf->device_lock);
-+ if (!atomic_read(&sh->count)) {
-+ if (!test_bit(STRIPE_HANDLE, &sh->state))
-+ atomic_inc(&conf->active_stripes);
-+ BUG_ON(list_empty(&sh->lru) &&
-+ !test_bit(STRIPE_EXPANDING, &sh->state));
-+ list_del_init(&sh->lru);
-+ if (sh->group) {
-+ sh->group->stripes_cnt--;
-+ sh->group = NULL;
-+ }
-+ }
-+ atomic_inc(&sh->count);
-+ spin_unlock(&conf->device_lock);
-+ }
-+ } while (sh == NULL);
-+
-+ spin_unlock_irq(conf->hash_locks + hash);
-+ return sh;
-+}
-+
-+static bool is_full_stripe_write(struct stripe_head *sh)
-+{
-+ BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
-+ return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
-+}
-+
-+static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
-+{
-+ local_irq_disable();
-+ if (sh1 > sh2) {
-+ spin_lock(&sh2->stripe_lock);
-+ spin_lock_nested(&sh1->stripe_lock, 1);
-+ } else {
-+ spin_lock(&sh1->stripe_lock);
-+ spin_lock_nested(&sh2->stripe_lock, 1);
-+ }
-+}
-+
-+static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
-+{
-+ spin_unlock(&sh1->stripe_lock);
-+ spin_unlock(&sh2->stripe_lock);
-+ local_irq_enable();
-+}
-+
-+/* Only freshly new full stripe normal write stripe can be added to a batch list */
-+static bool stripe_can_batch(struct stripe_head *sh)
-+{
-+ return test_bit(STRIPE_BATCH_READY, &sh->state) &&
-+ !test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&
-+ is_full_stripe_write(sh);
-+}
-+
-+/* we only do back search */
-+static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)
-+{
-+ struct stripe_head *head;
-+ sector_t head_sector, tmp_sec;
-+ int hash;
-+ int dd_idx;
-+
-+ if (!stripe_can_batch(sh))
-+ return;
-+ /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
-+ tmp_sec = sh->sector;
-+ if (!sector_div(tmp_sec, conf->chunk_sectors))
-+ return;
-+ head_sector = sh->sector - STRIPE_SECTORS;
-+
-+ hash = stripe_hash_locks_hash(head_sector);
-+ spin_lock_irq(conf->hash_locks + hash);
-+ head = __find_stripe(conf, head_sector, conf->generation);
-+ if (head && !atomic_inc_not_zero(&head->count)) {
-+ spin_lock(&conf->device_lock);
-+ if (!atomic_read(&head->count)) {
-+ if (!test_bit(STRIPE_HANDLE, &head->state))
-+ atomic_inc(&conf->active_stripes);
-+ BUG_ON(list_empty(&head->lru) &&
-+ !test_bit(STRIPE_EXPANDING, &head->state));
-+ list_del_init(&head->lru);
-+ if (head->group) {
-+ head->group->stripes_cnt--;
-+ head->group = NULL;
-+ }
-+ }
-+ atomic_inc(&head->count);
-+ spin_unlock(&conf->device_lock);
-+ }
-+ spin_unlock_irq(conf->hash_locks + hash);
-+
-+ if (!head)
-+ return;
-+ if (!stripe_can_batch(head))
-+ goto out;
-+
-+ lock_two_stripes(head, sh);
-+ /* clear_batch_ready clear the flag */
-+ if (!stripe_can_batch(head) || !stripe_can_batch(sh))
-+ goto unlock_out;
-+
-+ if (sh->batch_head)
-+ goto unlock_out;
-+
-+ dd_idx = 0;
-+ while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
-+ dd_idx++;
-+ if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw)
-+ goto unlock_out;
-+
-+ if (head->batch_head) {
-+ spin_lock(&head->batch_head->batch_lock);
-+ /* This batch list is already running */
-+ if (!stripe_can_batch(head)) {
-+ spin_unlock(&head->batch_head->batch_lock);
-+ goto unlock_out;
-+ }
-+
-+ /*
-+ * at this point, head's BATCH_READY could be cleared, but we
-+ * can still add the stripe to batch list
-+ */
-+ list_add(&sh->batch_list, &head->batch_list);
-+ spin_unlock(&head->batch_head->batch_lock);
-+
-+ sh->batch_head = head->batch_head;
-+ } else {
-+ head->batch_head = head;
-+ sh->batch_head = head->batch_head;
-+ spin_lock(&head->batch_lock);
-+ list_add_tail(&sh->batch_list, &head->batch_list);
-+ spin_unlock(&head->batch_lock);
-+ }
-+
-+ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-+ if (atomic_dec_return(&conf->preread_active_stripes)
-+ < IO_THRESHOLD)
-+ md_wakeup_thread(conf->mddev->thread);
-+
-+ if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {
-+ int seq = sh->bm_seq;
-+ if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&
-+ sh->batch_head->bm_seq > seq)
-+ seq = sh->batch_head->bm_seq;
-+ set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);
-+ sh->batch_head->bm_seq = seq;
-+ }
-+
-+ atomic_inc(&sh->count);
-+unlock_out:
-+ unlock_two_stripes(head, sh);
-+out:
-+ release_stripe(head);
-+}
-+
-+/* Determine if 'data_offset' or 'new_data_offset' should be used
-+ * in this stripe_head.
-+ */
-+static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
-+{
-+ sector_t progress = conf->reshape_progress;
-+ /* Need a memory barrier to make sure we see the value
-+ * of conf->generation, or ->data_offset that was set before
-+ * reshape_progress was updated.
-+ */
-+ smp_rmb();
-+ if (progress == MaxSector)
-+ return 0;
-+ if (sh->generation == conf->generation - 1)
-+ return 0;
-+ /* We are in a reshape, and this is a new-generation stripe,
-+ * so use new_data_offset.
-+ */
-+ return 1;
-+}
-+
-+static void
-+raid5_end_read_request(struct bio *bi, int error);
-+static void
-+raid5_end_write_request(struct bio *bi, int error);
-+
-+static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
-+{
-+ struct r5conf *conf = sh->raid_conf;
-+ int i, disks = sh->disks;
-+ struct stripe_head *head_sh = sh;
-+
-+ might_sleep();
-+
-+ for (i = disks; i--; ) {
-+ int rw;
-+ int replace_only = 0;
-+ struct bio *bi, *rbi;
-+ struct md_rdev *rdev, *rrdev = NULL;
-+
-+ sh = head_sh;
-+ if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
-+ if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
-+ rw = WRITE_FUA;
-+ else
-+ rw = WRITE;
-+ if (test_bit(R5_Discard, &sh->dev[i].flags))
-+ rw |= REQ_DISCARD;
-+ } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
-+ rw = READ;
-+ else if (test_and_clear_bit(R5_WantReplace,
-+ &sh->dev[i].flags)) {
-+ rw = WRITE;
-+ replace_only = 1;
-+ } else
-+ continue;
-+ if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
-+ rw |= REQ_SYNC;
-+
-+again:
-+ bi = &sh->dev[i].req;
-+ rbi = &sh->dev[i].rreq; /* For writing to replacement */
-+
-+ rcu_read_lock();
-+ rrdev = rcu_dereference(conf->disks[i].replacement);
-+ smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
-+ rdev = rcu_dereference(conf->disks[i].rdev);
-+ if (!rdev) {
-+ rdev = rrdev;
-+ rrdev = NULL;
-+ }
-+ if (rw & WRITE) {
-+ if (replace_only)
-+ rdev = NULL;
-+ if (rdev == rrdev)
-+ /* We raced and saw duplicates */
-+ rrdev = NULL;
-+ } else {
-+ if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)
-+ rdev = rrdev;
-+ rrdev = NULL;
-+ }
-+
-+ if (rdev && test_bit(Faulty, &rdev->flags))
-+ rdev = NULL;
-+ if (rdev)
-+ atomic_inc(&rdev->nr_pending);
-+ if (rrdev && test_bit(Faulty, &rrdev->flags))
-+ rrdev = NULL;
-+ if (rrdev)
-+ atomic_inc(&rrdev->nr_pending);
-+ rcu_read_unlock();
-+
-+ /* We have already checked bad blocks for reads. Now
-+ * need to check for writes. We never accept write errors
-+ * on the replacement, so we don't to check rrdev.
-+ */
-+ while ((rw & WRITE) && rdev &&
-+ test_bit(WriteErrorSeen, &rdev->flags)) {
-+ sector_t first_bad;
-+ int bad_sectors;
-+ int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
-+ &first_bad, &bad_sectors);
-+ if (!bad)
-+ break;
-+
-+ if (bad < 0) {
-+ set_bit(BlockedBadBlocks, &rdev->flags);
-+ if (!conf->mddev->external &&
-+ conf->mddev->flags) {
-+ /* It is very unlikely, but we might
-+ * still need to write out the
-+ * bad block log - better give it
-+ * a chance*/
-+ md_check_recovery(conf->mddev);
-+ }
-+ /*
-+ * Because md_wait_for_blocked_rdev
-+ * will dec nr_pending, we must
-+ * increment it first.
-+ */
-+ atomic_inc(&rdev->nr_pending);
-+ md_wait_for_blocked_rdev(rdev, conf->mddev);
-+ } else {
-+ /* Acknowledged bad block - skip the write */
-+ rdev_dec_pending(rdev, conf->mddev);
-+ rdev = NULL;
-+ }
-+ }
-+
-+ if (rdev) {
-+ if (s->syncing || s->expanding || s->expanded
-+ || s->replacing)
-+ md_sync_acct(rdev->bdev, STRIPE_SECTORS);
-+
-+ set_bit(STRIPE_IO_STARTED, &sh->state);
-+
-+ bio_reset(bi);
-+ bi->bi_bdev = rdev->bdev;
-+ bi->bi_rw = rw;
-+ bi->bi_end_io = (rw & WRITE)
-+ ? raid5_end_write_request
-+ : raid5_end_read_request;
-+ bi->bi_private = sh;
-+
-+ pr_debug("%s: for %llu schedule op %ld on disc %d\n",
-+ __func__, (unsigned long long)sh->sector,
-+ bi->bi_rw, i);
-+ atomic_inc(&sh->count);
-+ if (sh != head_sh)
-+ atomic_inc(&head_sh->count);
-+ if (use_new_offset(conf, sh))
-+ bi->bi_iter.bi_sector = (sh->sector
-+ + rdev->new_data_offset);
-+ else
-+ bi->bi_iter.bi_sector = (sh->sector
-+ + rdev->data_offset);
-+ if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))
-+ bi->bi_rw |= REQ_NOMERGE;
-+
-+ if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
-+ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
-+ sh->dev[i].vec.bv_page = sh->dev[i].page;
-+ bi->bi_vcnt = 1;
-+ bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
-+ bi->bi_io_vec[0].bv_offset = 0;
-+ bi->bi_iter.bi_size = STRIPE_SIZE;
-+ /*
-+ * If this is discard request, set bi_vcnt 0. We don't
-+ * want to confuse SCSI because SCSI will replace payload
-+ */
-+ if (rw & REQ_DISCARD)
-+ bi->bi_vcnt = 0;
-+ if (rrdev)
-+ set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
-+
-+ if (conf->mddev->gendisk)
-+ trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
-+ bi, disk_devt(conf->mddev->gendisk),
-+ sh->dev[i].sector);
-+ generic_make_request(bi);
-+ }
-+ if (rrdev) {
-+ if (s->syncing || s->expanding || s->expanded
-+ || s->replacing)
-+ md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
-+
-+ set_bit(STRIPE_IO_STARTED, &sh->state);
-+
-+ bio_reset(rbi);
-+ rbi->bi_bdev = rrdev->bdev;
-+ rbi->bi_rw = rw;
-+ BUG_ON(!(rw & WRITE));
-+ rbi->bi_end_io = raid5_end_write_request;
-+ rbi->bi_private = sh;
-+
-+ pr_debug("%s: for %llu schedule op %ld on "
-+ "replacement disc %d\n",
-+ __func__, (unsigned long long)sh->sector,
-+ rbi->bi_rw, i);
-+ atomic_inc(&sh->count);
-+ if (sh != head_sh)
-+ atomic_inc(&head_sh->count);
-+ if (use_new_offset(conf, sh))
-+ rbi->bi_iter.bi_sector = (sh->sector
-+ + rrdev->new_data_offset);
-+ else
-+ rbi->bi_iter.bi_sector = (sh->sector
-+ + rrdev->data_offset);
-+ if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
-+ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
-+ sh->dev[i].rvec.bv_page = sh->dev[i].page;
-+ rbi->bi_vcnt = 1;
-+ rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
-+ rbi->bi_io_vec[0].bv_offset = 0;
-+ rbi->bi_iter.bi_size = STRIPE_SIZE;
-+ /*
-+ * If this is discard request, set bi_vcnt 0. We don't
-+ * want to confuse SCSI because SCSI will replace payload
-+ */
-+ if (rw & REQ_DISCARD)
-+ rbi->bi_vcnt = 0;
-+ if (conf->mddev->gendisk)
-+ trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
-+ rbi, disk_devt(conf->mddev->gendisk),
-+ sh->dev[i].sector);
-+ generic_make_request(rbi);
-+ }
-+ if (!rdev && !rrdev) {
-+ if (rw & WRITE)
-+ set_bit(STRIPE_DEGRADED, &sh->state);
-+ pr_debug("skip op %ld on disc %d for sector %llu\n",
-+ bi->bi_rw, i, (unsigned long long)sh->sector);
-+ clear_bit(R5_LOCKED, &sh->dev[i].flags);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ }
-+
-+ if (!head_sh->batch_head)
-+ continue;
-+ sh = list_first_entry(&sh->batch_list, struct stripe_head,
-+ batch_list);
-+ if (sh != head_sh)
-+ goto again;
-+ }
-+}
-+
-+static struct dma_async_tx_descriptor *
-+async_copy_data(int frombio, struct bio *bio, struct page **page,
-+ sector_t sector, struct dma_async_tx_descriptor *tx,
-+ struct stripe_head *sh)
-+{
-+ struct bio_vec bvl;
-+ struct bvec_iter iter;
-+ struct page *bio_page;
-+ int page_offset;
-+ struct async_submit_ctl submit;
-+ enum async_tx_flags flags = 0;
-+
-+ if (bio->bi_iter.bi_sector >= sector)
-+ page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
-+ else
-+ page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
-+
-+ if (frombio)
-+ flags |= ASYNC_TX_FENCE;
-+ init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
-+
-+ bio_for_each_segment(bvl, bio, iter) {
-+ int len = bvl.bv_len;
-+ int clen;
-+ int b_offset = 0;
-+
-+ if (page_offset < 0) {
-+ b_offset = -page_offset;
-+ page_offset += b_offset;
-+ len -= b_offset;
-+ }
-+
-+ if (len > 0 && page_offset + len > STRIPE_SIZE)
-+ clen = STRIPE_SIZE - page_offset;
-+ else
-+ clen = len;
-+
-+ if (clen > 0) {
-+ b_offset += bvl.bv_offset;
-+ bio_page = bvl.bv_page;
-+ if (frombio) {
-+ if (sh->raid_conf->skip_copy &&
-+ b_offset == 0 && page_offset == 0 &&
-+ clen == STRIPE_SIZE)
-+ *page = bio_page;
-+ else
-+ tx = async_memcpy(*page, bio_page, page_offset,
-+ b_offset, clen, &submit);
-+ } else
-+ tx = async_memcpy(bio_page, *page, b_offset,
-+ page_offset, clen, &submit);
-+ }
-+ /* chain the operations */
-+ submit.depend_tx = tx;
-+
-+ if (clen < len) /* hit end of page */
-+ break;
-+ page_offset += len;
-+ }
-+
-+ return tx;
-+}
-+
-+static void ops_complete_biofill(void *stripe_head_ref)
-+{
-+ struct stripe_head *sh = stripe_head_ref;
-+ struct bio *return_bi = NULL;
-+ int i;
-+
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ /* clear completed biofills */
-+ for (i = sh->disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+
-+ /* acknowledge completion of a biofill operation */
-+ /* and check if we need to reply to a read request,
-+ * new R5_Wantfill requests are held off until
-+ * !STRIPE_BIOFILL_RUN
-+ */
-+ if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
-+ struct bio *rbi, *rbi2;
-+
-+ BUG_ON(!dev->read);
-+ rbi = dev->read;
-+ dev->read = NULL;
-+ while (rbi && rbi->bi_iter.bi_sector <
-+ dev->sector + STRIPE_SECTORS) {
-+ rbi2 = r5_next_bio(rbi, dev->sector);
-+ if (!raid5_dec_bi_active_stripes(rbi)) {
-+ rbi->bi_next = return_bi;
-+ return_bi = rbi;
-+ }
-+ rbi = rbi2;
-+ }
-+ }
-+ }
-+ clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
-+
-+ return_io(return_bi);
-+
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ release_stripe(sh);
-+}
-+
-+static void ops_run_biofill(struct stripe_head *sh)
-+{
-+ struct dma_async_tx_descriptor *tx = NULL;
-+ struct async_submit_ctl submit;
-+ int i;
-+
-+ BUG_ON(sh->batch_head);
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ for (i = sh->disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+ if (test_bit(R5_Wantfill, &dev->flags)) {
-+ struct bio *rbi;
-+ spin_lock_irq(&sh->stripe_lock);
-+ dev->read = rbi = dev->toread;
-+ dev->toread = NULL;
-+ spin_unlock_irq(&sh->stripe_lock);
-+ while (rbi && rbi->bi_iter.bi_sector <
-+ dev->sector + STRIPE_SECTORS) {
-+ tx = async_copy_data(0, rbi, &dev->page,
-+ dev->sector, tx, sh);
-+ rbi = r5_next_bio(rbi, dev->sector);
-+ }
-+ }
-+ }
-+
-+ atomic_inc(&sh->count);
-+ init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
-+ async_trigger_callback(&submit);
-+}
-+
-+static void mark_target_uptodate(struct stripe_head *sh, int target)
-+{
-+ struct r5dev *tgt;
-+
-+ if (target < 0)
-+ return;
-+
-+ tgt = &sh->dev[target];
-+ set_bit(R5_UPTODATE, &tgt->flags);
-+ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
-+ clear_bit(R5_Wantcompute, &tgt->flags);
-+}
-+
-+static void ops_complete_compute(void *stripe_head_ref)
-+{
-+ struct stripe_head *sh = stripe_head_ref;
-+
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ /* mark the computed target(s) as uptodate */
-+ mark_target_uptodate(sh, sh->ops.target);
-+ mark_target_uptodate(sh, sh->ops.target2);
-+
-+ clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
-+ if (sh->check_state == check_state_compute_run)
-+ sh->check_state = check_state_compute_result;
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ release_stripe(sh);
-+}
-+
-+/* return a pointer to the address conversion region of the scribble buffer */
-+static addr_conv_t *to_addr_conv(struct stripe_head *sh,
-+ struct raid5_percpu *percpu, int i)
-+{
-+ void *addr;
-+
-+ addr = flex_array_get(percpu->scribble, i);
-+ return addr + sizeof(struct page *) * (sh->disks + 2);
-+}
-+
-+/* return a pointer to the address conversion region of the scribble buffer */
-+static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
-+{
-+ void *addr;
-+
-+ addr = flex_array_get(percpu->scribble, i);
-+ return addr;
-+}
-+
-+static struct dma_async_tx_descriptor *
-+ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
-+{
-+ int disks = sh->disks;
-+ struct page **xor_srcs = to_addr_page(percpu, 0);
-+ int target = sh->ops.target;
-+ struct r5dev *tgt = &sh->dev[target];
-+ struct page *xor_dest = tgt->page;
-+ int count = 0;
-+ struct dma_async_tx_descriptor *tx;
-+ struct async_submit_ctl submit;
-+ int i;
-+
-+ BUG_ON(sh->batch_head);
-+
-+ pr_debug("%s: stripe %llu block: %d\n",
-+ __func__, (unsigned long long)sh->sector, target);
-+ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
-+
-+ for (i = disks; i--; )
-+ if (i != target)
-+ xor_srcs[count++] = sh->dev[i].page;
-+
-+ atomic_inc(&sh->count);
-+
-+ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
-+ ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
-+ if (unlikely(count == 1))
-+ tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
-+ else
-+ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
-+
-+ return tx;
-+}
-+
-+/* set_syndrome_sources - populate source buffers for gen_syndrome
-+ * @srcs - (struct page *) array of size sh->disks
-+ * @sh - stripe_head to parse
-+ *
-+ * Populates srcs in proper layout order for the stripe and returns the
-+ * 'count' of sources to be used in a call to async_gen_syndrome. The P
-+ * destination buffer is recorded in srcs[count] and the Q destination
-+ * is recorded in srcs[count+1]].
-+ */
-+static int set_syndrome_sources(struct page **srcs,
-+ struct stripe_head *sh,
-+ int srctype)
-+{
-+ int disks = sh->disks;
-+ int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
-+ int d0_idx = raid6_d0(sh);
-+ int count;
-+ int i;
-+
-+ for (i = 0; i < disks; i++)
-+ srcs[i] = NULL;
-+
-+ count = 0;
-+ i = d0_idx;
-+ do {
-+ int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
-+ struct r5dev *dev = &sh->dev[i];
-+
-+ if (i == sh->qd_idx || i == sh->pd_idx ||
-+ (srctype == SYNDROME_SRC_ALL) ||
-+ (srctype == SYNDROME_SRC_WANT_DRAIN &&
-+ test_bit(R5_Wantdrain, &dev->flags)) ||
-+ (srctype == SYNDROME_SRC_WRITTEN &&
-+ dev->written))
-+ srcs[slot] = sh->dev[i].page;
-+ i = raid6_next_disk(i, disks);
-+ } while (i != d0_idx);
-+
-+ return syndrome_disks;
-+}
-+
-+static struct dma_async_tx_descriptor *
-+ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
-+{
-+ int disks = sh->disks;
-+ struct page **blocks = to_addr_page(percpu, 0);
-+ int target;
-+ int qd_idx = sh->qd_idx;
-+ struct dma_async_tx_descriptor *tx;
-+ struct async_submit_ctl submit;
-+ struct r5dev *tgt;
-+ struct page *dest;
-+ int i;
-+ int count;
-+
-+ BUG_ON(sh->batch_head);
-+ if (sh->ops.target < 0)
-+ target = sh->ops.target2;
-+ else if (sh->ops.target2 < 0)
-+ target = sh->ops.target;
-+ else
-+ /* we should only have one valid target */
-+ BUG();
-+ BUG_ON(target < 0);
-+ pr_debug("%s: stripe %llu block: %d\n",
-+ __func__, (unsigned long long)sh->sector, target);
-+
-+ tgt = &sh->dev[target];
-+ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
-+ dest = tgt->page;
-+
-+ atomic_inc(&sh->count);
-+
-+ if (target == qd_idx) {
-+ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
-+ blocks[count] = NULL; /* regenerating p is not necessary */
-+ BUG_ON(blocks[count+1] != dest); /* q should already be set */
-+ init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
-+ ops_complete_compute, sh,
-+ to_addr_conv(sh, percpu, 0));
-+ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
-+ } else {
-+ /* Compute any data- or p-drive using XOR */
-+ count = 0;
-+ for (i = disks; i-- ; ) {
-+ if (i == target || i == qd_idx)
-+ continue;
-+ blocks[count++] = sh->dev[i].page;
-+ }
-+
-+ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
-+ NULL, ops_complete_compute, sh,
-+ to_addr_conv(sh, percpu, 0));
-+ tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
-+ }
-+
-+ return tx;
-+}
-+
-+static struct dma_async_tx_descriptor *
-+ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
-+{
-+ int i, count, disks = sh->disks;
-+ int syndrome_disks = sh->ddf_layout ? disks : disks-2;
-+ int d0_idx = raid6_d0(sh);
-+ int faila = -1, failb = -1;
-+ int target = sh->ops.target;
-+ int target2 = sh->ops.target2;
-+ struct r5dev *tgt = &sh->dev[target];
-+ struct r5dev *tgt2 = &sh->dev[target2];
-+ struct dma_async_tx_descriptor *tx;
-+ struct page **blocks = to_addr_page(percpu, 0);
-+ struct async_submit_ctl submit;
-+
-+ BUG_ON(sh->batch_head);
-+ pr_debug("%s: stripe %llu block1: %d block2: %d\n",
-+ __func__, (unsigned long long)sh->sector, target, target2);
-+ BUG_ON(target < 0 || target2 < 0);
-+ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
-+ BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
-+
-+ /* we need to open-code set_syndrome_sources to handle the
-+ * slot number conversion for 'faila' and 'failb'
-+ */
-+ for (i = 0; i < disks ; i++)
-+ blocks[i] = NULL;
-+ count = 0;
-+ i = d0_idx;
-+ do {
-+ int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
-+
-+ blocks[slot] = sh->dev[i].page;
-+
-+ if (i == target)
-+ faila = slot;
-+ if (i == target2)
-+ failb = slot;
-+ i = raid6_next_disk(i, disks);
-+ } while (i != d0_idx);
-+
-+ BUG_ON(faila == failb);
-+ if (failb < faila)
-+ swap(faila, failb);
-+ pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
-+ __func__, (unsigned long long)sh->sector, faila, failb);
-+
-+ atomic_inc(&sh->count);
-+
-+ if (failb == syndrome_disks+1) {
-+ /* Q disk is one of the missing disks */
-+ if (faila == syndrome_disks) {
-+ /* Missing P+Q, just recompute */
-+ init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
-+ ops_complete_compute, sh,
-+ to_addr_conv(sh, percpu, 0));
-+ return async_gen_syndrome(blocks, 0, syndrome_disks+2,
-+ STRIPE_SIZE, &submit);
-+ } else {
-+ struct page *dest;
-+ int data_target;
-+ int qd_idx = sh->qd_idx;
-+
-+ /* Missing D+Q: recompute D from P, then recompute Q */
-+ if (target == qd_idx)
-+ data_target = target2;
-+ else
-+ data_target = target;
-+
-+ count = 0;
-+ for (i = disks; i-- ; ) {
-+ if (i == data_target || i == qd_idx)
-+ continue;
-+ blocks[count++] = sh->dev[i].page;
-+ }
-+ dest = sh->dev[data_target].page;
-+ init_async_submit(&submit,
-+ ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
-+ NULL, NULL, NULL,
-+ to_addr_conv(sh, percpu, 0));
-+ tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
-+ &submit);
-+
-+ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
-+ init_async_submit(&submit, ASYNC_TX_FENCE, tx,
-+ ops_complete_compute, sh,
-+ to_addr_conv(sh, percpu, 0));
-+ return async_gen_syndrome(blocks, 0, count+2,
-+ STRIPE_SIZE, &submit);
-+ }
-+ } else {
-+ init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
-+ ops_complete_compute, sh,
-+ to_addr_conv(sh, percpu, 0));
-+ if (failb == syndrome_disks) {
-+ /* We're missing D+P. */
-+ return async_raid6_datap_recov(syndrome_disks+2,
-+ STRIPE_SIZE, faila,
-+ blocks, &submit);
-+ } else {
-+ /* We're missing D+D. */
-+ return async_raid6_2data_recov(syndrome_disks+2,
-+ STRIPE_SIZE, faila, failb,
-+ blocks, &submit);
-+ }
-+ }
-+}
-+
-+static void ops_complete_prexor(void *stripe_head_ref)
-+{
-+ struct stripe_head *sh = stripe_head_ref;
-+
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+}
-+
-+static struct dma_async_tx_descriptor *
-+ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
-+ struct dma_async_tx_descriptor *tx)
-+{
-+ int disks = sh->disks;
-+ struct page **xor_srcs = to_addr_page(percpu, 0);
-+ int count = 0, pd_idx = sh->pd_idx, i;
-+ struct async_submit_ctl submit;
-+
-+ /* existing parity data subtracted */
-+ struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
-+
-+ BUG_ON(sh->batch_head);
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+ /* Only process blocks that are known to be uptodate */
-+ if (test_bit(R5_Wantdrain, &dev->flags))
-+ xor_srcs[count++] = dev->page;
-+ }
-+
-+ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
-+ ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
-+ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
-+
-+ return tx;
-+}
-+
-+static struct dma_async_tx_descriptor *
-+ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
-+ struct dma_async_tx_descriptor *tx)
-+{
-+ struct page **blocks = to_addr_page(percpu, 0);
-+ int count;
-+ struct async_submit_ctl submit;
-+
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN);
-+
-+ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
-+ ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
-+ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
-+
-+ return tx;
-+}
-+
-+static struct dma_async_tx_descriptor *
-+ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
-+{
-+ int disks = sh->disks;
-+ int i;
-+ struct stripe_head *head_sh = sh;
-+
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev;
-+ struct bio *chosen;
-+
-+ sh = head_sh;
-+ if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {
-+ struct bio *wbi;
-+
-+again:
-+ dev = &sh->dev[i];
-+ spin_lock_irq(&sh->stripe_lock);
-+ chosen = dev->towrite;
-+ dev->towrite = NULL;
-+ sh->overwrite_disks = 0;
-+ BUG_ON(dev->written);
-+ wbi = dev->written = chosen;
-+ spin_unlock_irq(&sh->stripe_lock);
-+ WARN_ON(dev->page != dev->orig_page);
-+
-+ while (wbi && wbi->bi_iter.bi_sector <
-+ dev->sector + STRIPE_SECTORS) {
-+ if (wbi->bi_rw & REQ_FUA)
-+ set_bit(R5_WantFUA, &dev->flags);
-+ if (wbi->bi_rw & REQ_SYNC)
-+ set_bit(R5_SyncIO, &dev->flags);
-+ if (wbi->bi_rw & REQ_DISCARD)
-+ set_bit(R5_Discard, &dev->flags);
-+ else {
-+ tx = async_copy_data(1, wbi, &dev->page,
-+ dev->sector, tx, sh);
-+ if (dev->page != dev->orig_page) {
-+ set_bit(R5_SkipCopy, &dev->flags);
-+ clear_bit(R5_UPTODATE, &dev->flags);
-+ clear_bit(R5_OVERWRITE, &dev->flags);
-+ }
-+ }
-+ wbi = r5_next_bio(wbi, dev->sector);
-+ }
-+
-+ if (head_sh->batch_head) {
-+ sh = list_first_entry(&sh->batch_list,
-+ struct stripe_head,
-+ batch_list);
-+ if (sh == head_sh)
-+ continue;
-+ goto again;
-+ }
-+ }
-+ }
-+
-+ return tx;
-+}
-+
-+static void ops_complete_reconstruct(void *stripe_head_ref)
-+{
-+ struct stripe_head *sh = stripe_head_ref;
-+ int disks = sh->disks;
-+ int pd_idx = sh->pd_idx;
-+ int qd_idx = sh->qd_idx;
-+ int i;
-+ bool fua = false, sync = false, discard = false;
-+
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ for (i = disks; i--; ) {
-+ fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
-+ sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
-+ discard |= test_bit(R5_Discard, &sh->dev[i].flags);
-+ }
-+
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+
-+ if (dev->written || i == pd_idx || i == qd_idx) {
-+ if (!discard && !test_bit(R5_SkipCopy, &dev->flags))
-+ set_bit(R5_UPTODATE, &dev->flags);
-+ if (fua)
-+ set_bit(R5_WantFUA, &dev->flags);
-+ if (sync)
-+ set_bit(R5_SyncIO, &dev->flags);
-+ }
-+ }
-+
-+ if (sh->reconstruct_state == reconstruct_state_drain_run)
-+ sh->reconstruct_state = reconstruct_state_drain_result;
-+ else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
-+ sh->reconstruct_state = reconstruct_state_prexor_drain_result;
-+ else {
-+ BUG_ON(sh->reconstruct_state != reconstruct_state_run);
-+ sh->reconstruct_state = reconstruct_state_result;
-+ }
-+
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ release_stripe(sh);
-+}
-+
-+static void
-+ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
-+ struct dma_async_tx_descriptor *tx)
-+{
-+ int disks = sh->disks;
-+ struct page **xor_srcs;
-+ struct async_submit_ctl submit;
-+ int count, pd_idx = sh->pd_idx, i;
-+ struct page *xor_dest;
-+ int prexor = 0;
-+ unsigned long flags;
-+ int j = 0;
-+ struct stripe_head *head_sh = sh;
-+ int last_stripe;
-+
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ for (i = 0; i < sh->disks; i++) {
-+ if (pd_idx == i)
-+ continue;
-+ if (!test_bit(R5_Discard, &sh->dev[i].flags))
-+ break;
-+ }
-+ if (i >= sh->disks) {
-+ atomic_inc(&sh->count);
-+ set_bit(R5_Discard, &sh->dev[pd_idx].flags);
-+ ops_complete_reconstruct(sh);
-+ return;
-+ }
-+again:
-+ count = 0;
-+ xor_srcs = to_addr_page(percpu, j);
-+ /* check if prexor is active which means only process blocks
-+ * that are part of a read-modify-write (written)
-+ */
-+ if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
-+ prexor = 1;
-+ xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+ if (head_sh->dev[i].written)
-+ xor_srcs[count++] = dev->page;
-+ }
-+ } else {
-+ xor_dest = sh->dev[pd_idx].page;
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+ if (i != pd_idx)
-+ xor_srcs[count++] = dev->page;
-+ }
-+ }
-+
-+ /* 1/ if we prexor'd then the dest is reused as a source
-+ * 2/ if we did not prexor then we are redoing the parity
-+ * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
-+ * for the synchronous xor case
-+ */
-+ last_stripe = !head_sh->batch_head ||
-+ list_first_entry(&sh->batch_list,
-+ struct stripe_head, batch_list) == head_sh;
-+ if (last_stripe) {
-+ flags = ASYNC_TX_ACK |
-+ (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
-+
-+ atomic_inc(&head_sh->count);
-+ init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
-+ to_addr_conv(sh, percpu, j));
-+ } else {
-+ flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
-+ init_async_submit(&submit, flags, tx, NULL, NULL,
-+ to_addr_conv(sh, percpu, j));
-+ }
-+
-+ if (unlikely(count == 1))
-+ tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
-+ else
-+ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
-+ if (!last_stripe) {
-+ j++;
-+ sh = list_first_entry(&sh->batch_list, struct stripe_head,
-+ batch_list);
-+ goto again;
-+ }
-+}
-+
-+static void
-+ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
-+ struct dma_async_tx_descriptor *tx)
-+{
-+ struct async_submit_ctl submit;
-+ struct page **blocks;
-+ int count, i, j = 0;
-+ struct stripe_head *head_sh = sh;
-+ int last_stripe;
-+ int synflags;
-+ unsigned long txflags;
-+
-+ pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
-+
-+ for (i = 0; i < sh->disks; i++) {
-+ if (sh->pd_idx == i || sh->qd_idx == i)
-+ continue;
-+ if (!test_bit(R5_Discard, &sh->dev[i].flags))
-+ break;
-+ }
-+ if (i >= sh->disks) {
-+ atomic_inc(&sh->count);
-+ set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
-+ set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
-+ ops_complete_reconstruct(sh);
-+ return;
-+ }
-+
-+again:
-+ blocks = to_addr_page(percpu, j);
-+
-+ if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
-+ synflags = SYNDROME_SRC_WRITTEN;
-+ txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
-+ } else {
-+ synflags = SYNDROME_SRC_ALL;
-+ txflags = ASYNC_TX_ACK;
-+ }
-+
-+ count = set_syndrome_sources(blocks, sh, synflags);
-+ last_stripe = !head_sh->batch_head ||
-+ list_first_entry(&sh->batch_list,
-+ struct stripe_head, batch_list) == head_sh;
-+
-+ if (last_stripe) {
-+ atomic_inc(&head_sh->count);
-+ init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
-+ head_sh, to_addr_conv(sh, percpu, j));
-+ } else
-+ init_async_submit(&submit, 0, tx, NULL, NULL,
-+ to_addr_conv(sh, percpu, j));
-+ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
-+ if (!last_stripe) {
-+ j++;
-+ sh = list_first_entry(&sh->batch_list, struct stripe_head,
-+ batch_list);
-+ goto again;
-+ }
-+}
-+
-+static void ops_complete_check(void *stripe_head_ref)
-+{
-+ struct stripe_head *sh = stripe_head_ref;
-+
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ sh->check_state = check_state_check_result;
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ release_stripe(sh);
-+}
-+
-+static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
-+{
-+ int disks = sh->disks;
-+ int pd_idx = sh->pd_idx;
-+ int qd_idx = sh->qd_idx;
-+ struct page *xor_dest;
-+ struct page **xor_srcs = to_addr_page(percpu, 0);
-+ struct dma_async_tx_descriptor *tx;
-+ struct async_submit_ctl submit;
-+ int count;
-+ int i;
-+
-+ pr_debug("%s: stripe %llu\n", __func__,
-+ (unsigned long long)sh->sector);
-+
-+ BUG_ON(sh->batch_head);
-+ count = 0;
-+ xor_dest = sh->dev[pd_idx].page;
-+ xor_srcs[count++] = xor_dest;
-+ for (i = disks; i--; ) {
-+ if (i == pd_idx || i == qd_idx)
-+ continue;
-+ xor_srcs[count++] = sh->dev[i].page;
-+ }
-+
-+ init_async_submit(&submit, 0, NULL, NULL, NULL,
-+ to_addr_conv(sh, percpu, 0));
-+ tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
-+ &sh->ops.zero_sum_result, &submit);
-+
-+ atomic_inc(&sh->count);
-+ init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
-+ tx = async_trigger_callback(&submit);
-+}
-+
-+static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
-+{
-+ struct page **srcs = to_addr_page(percpu, 0);
-+ struct async_submit_ctl submit;
-+ int count;
-+
-+ pr_debug("%s: stripe %llu checkp: %d\n", __func__,
-+ (unsigned long long)sh->sector, checkp);
-+
-+ BUG_ON(sh->batch_head);
-+ count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL);
-+ if (!checkp)
-+ srcs[count] = NULL;
-+
-+ atomic_inc(&sh->count);
-+ init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
-+ sh, to_addr_conv(sh, percpu, 0));
-+ async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
-+ &sh->ops.zero_sum_result, percpu->spare_page, &submit);
-+}
-+
-+static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
-+{
-+ int overlap_clear = 0, i, disks = sh->disks;
-+ struct dma_async_tx_descriptor *tx = NULL;
-+ struct r5conf *conf = sh->raid_conf;
-+ int level = conf->level;
-+ struct raid5_percpu *percpu;
-+ unsigned long cpu;
-+
-+ cpu = get_cpu();
-+ percpu = per_cpu_ptr(conf->percpu, cpu);
-+ if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
-+ ops_run_biofill(sh);
-+ overlap_clear++;
-+ }
-+
-+ if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
-+ if (level < 6)
-+ tx = ops_run_compute5(sh, percpu);
-+ else {
-+ if (sh->ops.target2 < 0 || sh->ops.target < 0)
-+ tx = ops_run_compute6_1(sh, percpu);
-+ else
-+ tx = ops_run_compute6_2(sh, percpu);
-+ }
-+ /* terminate the chain if reconstruct is not set to be run */
-+ if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
-+ async_tx_ack(tx);
-+ }
-+
-+ if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
-+ if (level < 6)
-+ tx = ops_run_prexor5(sh, percpu, tx);
-+ else
-+ tx = ops_run_prexor6(sh, percpu, tx);
-+ }
-+
-+ if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
-+ tx = ops_run_biodrain(sh, tx);
-+ overlap_clear++;
-+ }
-+
-+ if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
-+ if (level < 6)
-+ ops_run_reconstruct5(sh, percpu, tx);
-+ else
-+ ops_run_reconstruct6(sh, percpu, tx);
-+ }
-+
-+ if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
-+ if (sh->check_state == check_state_run)
-+ ops_run_check_p(sh, percpu);
-+ else if (sh->check_state == check_state_run_q)
-+ ops_run_check_pq(sh, percpu, 0);
-+ else if (sh->check_state == check_state_run_pq)
-+ ops_run_check_pq(sh, percpu, 1);
-+ else
-+ BUG();
-+ }
-+
-+ if (overlap_clear && !sh->batch_head)
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+ if (test_and_clear_bit(R5_Overlap, &dev->flags))
-+ wake_up(&sh->raid_conf->wait_for_overlap);
-+ }
-+ put_cpu();
-+}
-+
-+static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp)
-+{
-+ struct stripe_head *sh;
-+
-+ sh = kmem_cache_zalloc(sc, gfp);
-+ if (sh) {
-+ spin_lock_init(&sh->stripe_lock);
-+ spin_lock_init(&sh->batch_lock);
-+ INIT_LIST_HEAD(&sh->batch_list);
-+ INIT_LIST_HEAD(&sh->lru);
-+ atomic_set(&sh->count, 1);
-+ }
-+ return sh;
-+}
-+static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
-+{
-+ struct stripe_head *sh;
-+
-+ sh = alloc_stripe(conf->slab_cache, gfp);
-+ if (!sh)
-+ return 0;
-+
-+ sh->raid_conf = conf;
-+
-+ if (grow_buffers(sh, gfp)) {
-+ shrink_buffers(sh);
-+ kmem_cache_free(conf->slab_cache, sh);
-+ return 0;
-+ }
-+ sh->hash_lock_index =
-+ conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
-+ /* we just created an active stripe so... */
-+ atomic_inc(&conf->active_stripes);
-+
-+ release_stripe(sh);
-+ conf->max_nr_stripes++;
-+ return 1;
-+}
-+
-+static int grow_stripes(struct r5conf *conf, int num)
-+{
-+ struct kmem_cache *sc;
-+ int devs = max(conf->raid_disks, conf->previous_raid_disks);
-+
-+ if (conf->mddev->gendisk)
-+ sprintf(conf->cache_name[0],
-+ "raid%d-%s", conf->level, mdname(conf->mddev));
-+ else
-+ sprintf(conf->cache_name[0],
-+ "raid%d-%p", conf->level, conf->mddev);
-+ sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
-+
-+ conf->active_name = 0;
-+ sc = kmem_cache_create(conf->cache_name[conf->active_name],
-+ sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
-+ 0, 0, NULL);
-+ if (!sc)
-+ return 1;
-+ conf->slab_cache = sc;
-+ conf->pool_size = devs;
-+ while (num--)
-+ if (!grow_one_stripe(conf, GFP_KERNEL))
-+ return 1;
-+
-+ return 0;
-+}
-+
-+/**
-+ * scribble_len - return the required size of the scribble region
-+ * @num - total number of disks in the array
-+ *
-+ * The size must be enough to contain:
-+ * 1/ a struct page pointer for each device in the array +2
-+ * 2/ room to convert each entry in (1) to its corresponding dma
-+ * (dma_map_page()) or page (page_address()) address.
-+ *
-+ * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
-+ * calculate over all devices (not just the data blocks), using zeros in place
-+ * of the P and Q blocks.
-+ */
-+static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags)
-+{
-+ struct flex_array *ret;
-+ size_t len;
-+
-+ len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
-+ ret = flex_array_alloc(len, cnt, flags);
-+ if (!ret)
-+ return NULL;
-+ /* always prealloc all elements, so no locking is required */
-+ if (flex_array_prealloc(ret, 0, cnt, flags)) {
-+ flex_array_free(ret);
-+ return NULL;
-+ }
-+ return ret;
-+}
-+
-+static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
-+{
-+ unsigned long cpu;
-+ int err = 0;
-+
-+ mddev_suspend(conf->mddev);
-+ get_online_cpus();
-+ for_each_present_cpu(cpu) {
-+ struct raid5_percpu *percpu;
-+ struct flex_array *scribble;
-+
-+ percpu = per_cpu_ptr(conf->percpu, cpu);
-+ scribble = scribble_alloc(new_disks,
-+ new_sectors / STRIPE_SECTORS,
-+ GFP_NOIO);
-+
-+ if (scribble) {
-+ flex_array_free(percpu->scribble);
-+ percpu->scribble = scribble;
-+ } else {
-+ err = -ENOMEM;
-+ break;
-+ }
-+ }
-+ put_online_cpus();
-+ mddev_resume(conf->mddev);
-+ return err;
-+}
-+
-+static int resize_stripes(struct r5conf *conf, int newsize)
-+{
-+ /* Make all the stripes able to hold 'newsize' devices.
-+ * New slots in each stripe get 'page' set to a new page.
-+ *
-+ * This happens in stages:
-+ * 1/ create a new kmem_cache and allocate the required number of
-+ * stripe_heads.
-+ * 2/ gather all the old stripe_heads and transfer the pages across
-+ * to the new stripe_heads. This will have the side effect of
-+ * freezing the array as once all stripe_heads have been collected,
-+ * no IO will be possible. Old stripe heads are freed once their
-+ * pages have been transferred over, and the old kmem_cache is
-+ * freed when all stripes are done.
-+ * 3/ reallocate conf->disks to be suitable bigger. If this fails,
-+ * we simple return a failre status - no need to clean anything up.
-+ * 4/ allocate new pages for the new slots in the new stripe_heads.
-+ * If this fails, we don't bother trying the shrink the
-+ * stripe_heads down again, we just leave them as they are.
-+ * As each stripe_head is processed the new one is released into
-+ * active service.
-+ *
-+ * Once step2 is started, we cannot afford to wait for a write,
-+ * so we use GFP_NOIO allocations.
-+ */
-+ struct stripe_head *osh, *nsh;
-+ LIST_HEAD(newstripes);
-+ struct disk_info *ndisks;
-+ int err;
-+ struct kmem_cache *sc;
-+ int i;
-+ int hash, cnt;
-+
-+ if (newsize <= conf->pool_size)
-+ return 0; /* never bother to shrink */
-+
-+ err = md_allow_write(conf->mddev);
-+ if (err)
-+ return err;
-+
-+ /* Step 1 */
-+ sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
-+ sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
-+ 0, 0, NULL);
-+ if (!sc)
-+ return -ENOMEM;
-+
-+ /* Need to ensure auto-resizing doesn't interfere */
-+ mutex_lock(&conf->cache_size_mutex);
-+
-+ for (i = conf->max_nr_stripes; i; i--) {
-+ nsh = alloc_stripe(sc, GFP_KERNEL);
-+ if (!nsh)
-+ break;
-+
-+ nsh->raid_conf = conf;
-+ list_add(&nsh->lru, &newstripes);
-+ }
-+ if (i) {
-+ /* didn't get enough, give up */
-+ while (!list_empty(&newstripes)) {
-+ nsh = list_entry(newstripes.next, struct stripe_head, lru);
-+ list_del(&nsh->lru);
-+ kmem_cache_free(sc, nsh);
-+ }
-+ kmem_cache_destroy(sc);
-+ mutex_unlock(&conf->cache_size_mutex);
-+ return -ENOMEM;
-+ }
-+ /* Step 2 - Must use GFP_NOIO now.
-+ * OK, we have enough stripes, start collecting inactive
-+ * stripes and copying them over
-+ */
-+ hash = 0;
-+ cnt = 0;
-+ list_for_each_entry(nsh, &newstripes, lru) {
-+ lock_device_hash_lock(conf, hash);
-+ wait_event_cmd(conf->wait_for_stripe,
-+ !list_empty(conf->inactive_list + hash),
-+ unlock_device_hash_lock(conf, hash),
-+ lock_device_hash_lock(conf, hash));
-+ osh = get_free_stripe(conf, hash);
-+ unlock_device_hash_lock(conf, hash);
-+
-+ for(i=0; i<conf->pool_size; i++) {
-+ nsh->dev[i].page = osh->dev[i].page;
-+ nsh->dev[i].orig_page = osh->dev[i].page;
-+ }
-+ nsh->hash_lock_index = hash;
-+ kmem_cache_free(conf->slab_cache, osh);
-+ cnt++;
-+ if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
-+ !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
-+ hash++;
-+ cnt = 0;
-+ }
-+ }
-+ kmem_cache_destroy(conf->slab_cache);
-+
-+ /* Step 3.
-+ * At this point, we are holding all the stripes so the array
-+ * is completely stalled, so now is a good time to resize
-+ * conf->disks and the scribble region
-+ */
-+ ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
-+ if (ndisks) {
-+ for (i=0; i<conf->raid_disks; i++)
-+ ndisks[i] = conf->disks[i];
-+ kfree(conf->disks);
-+ conf->disks = ndisks;
-+ } else
-+ err = -ENOMEM;
-+
-+ mutex_unlock(&conf->cache_size_mutex);
-+ /* Step 4, return new stripes to service */
-+ while(!list_empty(&newstripes)) {
-+ nsh = list_entry(newstripes.next, struct stripe_head, lru);
-+ list_del_init(&nsh->lru);
-+
-+ for (i=conf->raid_disks; i < newsize; i++)
-+ if (nsh->dev[i].page == NULL) {
-+ struct page *p = alloc_page(GFP_NOIO);
-+ nsh->dev[i].page = p;
-+ nsh->dev[i].orig_page = p;
-+ if (!p)
-+ err = -ENOMEM;
-+ }
-+ release_stripe(nsh);
-+ }
-+ /* critical section pass, GFP_NOIO no longer needed */
-+
-+ conf->slab_cache = sc;
-+ conf->active_name = 1-conf->active_name;
-+ if (!err)
-+ conf->pool_size = newsize;
-+ return err;
-+}
-+
-+static int drop_one_stripe(struct r5conf *conf)
-+{
-+ struct stripe_head *sh;
-+ int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK;
-+
-+ spin_lock_irq(conf->hash_locks + hash);
-+ sh = get_free_stripe(conf, hash);
-+ spin_unlock_irq(conf->hash_locks + hash);
-+ if (!sh)
-+ return 0;
-+ BUG_ON(atomic_read(&sh->count));
-+ shrink_buffers(sh);
-+ kmem_cache_free(conf->slab_cache, sh);
-+ atomic_dec(&conf->active_stripes);
-+ conf->max_nr_stripes--;
-+ return 1;
-+}
-+
-+static void shrink_stripes(struct r5conf *conf)
-+{
-+ while (conf->max_nr_stripes &&
-+ drop_one_stripe(conf))
-+ ;
-+
-+ if (conf->slab_cache)
-+ kmem_cache_destroy(conf->slab_cache);
-+ conf->slab_cache = NULL;
-+}
-+
-+static void raid5_end_read_request(struct bio * bi, int error)
-+{
-+ struct stripe_head *sh = bi->bi_private;
-+ struct r5conf *conf = sh->raid_conf;
-+ int disks = sh->disks, i;
-+ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
-+ char b[BDEVNAME_SIZE];
-+ struct md_rdev *rdev = NULL;
-+ sector_t s;
-+
-+ for (i=0 ; i<disks; i++)
-+ if (bi == &sh->dev[i].req)
-+ break;
-+
-+ pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
-+ (unsigned long long)sh->sector, i, atomic_read(&sh->count),
-+ uptodate);
-+ if (i == disks) {
-+ BUG();
-+ return;
-+ }
-+ if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
-+ /* If replacement finished while this request was outstanding,
-+ * 'replacement' might be NULL already.
-+ * In that case it moved down to 'rdev'.
-+ * rdev is not removed until all requests are finished.
-+ */
-+ rdev = conf->disks[i].replacement;
-+ if (!rdev)
-+ rdev = conf->disks[i].rdev;
-+
-+ if (use_new_offset(conf, sh))
-+ s = sh->sector + rdev->new_data_offset;
-+ else
-+ s = sh->sector + rdev->data_offset;
-+ if (uptodate) {
-+ set_bit(R5_UPTODATE, &sh->dev[i].flags);
-+ if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
-+ /* Note that this cannot happen on a
-+ * replacement device. We just fail those on
-+ * any error
-+ */
-+ printk_ratelimited(
-+ KERN_INFO
-+ "md/raid:%s: read error corrected"
-+ " (%lu sectors at %llu on %s)\n",
-+ mdname(conf->mddev), STRIPE_SECTORS,
-+ (unsigned long long)s,
-+ bdevname(rdev->bdev, b));
-+ atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
-+ clear_bit(R5_ReadError, &sh->dev[i].flags);
-+ clear_bit(R5_ReWrite, &sh->dev[i].flags);
-+ } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
-+ clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
-+
-+ if (atomic_read(&rdev->read_errors))
-+ atomic_set(&rdev->read_errors, 0);
-+ } else {
-+ const char *bdn = bdevname(rdev->bdev, b);
-+ int retry = 0;
-+ int set_bad = 0;
-+
-+ clear_bit(R5_UPTODATE, &sh->dev[i].flags);
-+ atomic_inc(&rdev->read_errors);
-+ if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
-+ printk_ratelimited(
-+ KERN_WARNING
-+ "md/raid:%s: read error on replacement device "
-+ "(sector %llu on %s).\n",
-+ mdname(conf->mddev),
-+ (unsigned long long)s,
-+ bdn);
-+ else if (conf->mddev->degraded >= conf->max_degraded) {
-+ set_bad = 1;
-+ printk_ratelimited(
-+ KERN_WARNING
-+ "md/raid:%s: read error not correctable "
-+ "(sector %llu on %s).\n",
-+ mdname(conf->mddev),
-+ (unsigned long long)s,
-+ bdn);
-+ } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
-+ /* Oh, no!!! */
-+ set_bad = 1;
-+ printk_ratelimited(
-+ KERN_WARNING
-+ "md/raid:%s: read error NOT corrected!! "
-+ "(sector %llu on %s).\n",
-+ mdname(conf->mddev),
-+ (unsigned long long)s,
-+ bdn);
-+ } else if (atomic_read(&rdev->read_errors)
-+ > conf->max_nr_stripes)
-+ printk(KERN_WARNING
-+ "md/raid:%s: Too many read errors, failing device %s.\n",
-+ mdname(conf->mddev), bdn);
-+ else
-+ retry = 1;
-+ if (set_bad && test_bit(In_sync, &rdev->flags)
-+ && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
-+ retry = 1;
-+ if (retry)
-+ if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
-+ set_bit(R5_ReadError, &sh->dev[i].flags);
-+ clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
-+ } else
-+ set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
-+ else {
-+ clear_bit(R5_ReadError, &sh->dev[i].flags);
-+ clear_bit(R5_ReWrite, &sh->dev[i].flags);
-+ if (!(set_bad
-+ && test_bit(In_sync, &rdev->flags)
-+ && rdev_set_badblocks(
-+ rdev, sh->sector, STRIPE_SECTORS, 0)))
-+ md_error(conf->mddev, rdev);
-+ }
-+ }
-+ rdev_dec_pending(rdev, conf->mddev);
-+ clear_bit(R5_LOCKED, &sh->dev[i].flags);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ release_stripe(sh);
-+}
-+
-+static void raid5_end_write_request(struct bio *bi, int error)
-+{
-+ struct stripe_head *sh = bi->bi_private;
-+ struct r5conf *conf = sh->raid_conf;
-+ int disks = sh->disks, i;
-+ struct md_rdev *uninitialized_var(rdev);
-+ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
-+ sector_t first_bad;
-+ int bad_sectors;
-+ int replacement = 0;
-+
-+ for (i = 0 ; i < disks; i++) {
-+ if (bi == &sh->dev[i].req) {
-+ rdev = conf->disks[i].rdev;
-+ break;
-+ }
-+ if (bi == &sh->dev[i].rreq) {
-+ rdev = conf->disks[i].replacement;
-+ if (rdev)
-+ replacement = 1;
-+ else
-+ /* rdev was removed and 'replacement'
-+ * replaced it. rdev is not removed
-+ * until all requests are finished.
-+ */
-+ rdev = conf->disks[i].rdev;
-+ break;
-+ }
-+ }
-+ pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
-+ (unsigned long long)sh->sector, i, atomic_read(&sh->count),
-+ uptodate);
-+ if (i == disks) {
-+ BUG();
-+ return;
-+ }
-+
-+ if (replacement) {
-+ if (!uptodate)
-+ md_error(conf->mddev, rdev);
-+ else if (is_badblock(rdev, sh->sector,
-+ STRIPE_SECTORS,
-+ &first_bad, &bad_sectors))
-+ set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
-+ } else {
-+ if (!uptodate) {
-+ set_bit(STRIPE_DEGRADED, &sh->state);
-+ set_bit(WriteErrorSeen, &rdev->flags);
-+ set_bit(R5_WriteError, &sh->dev[i].flags);
-+ if (!test_and_set_bit(WantReplacement, &rdev->flags))
-+ set_bit(MD_RECOVERY_NEEDED,
-+ &rdev->mddev->recovery);
-+ } else if (is_badblock(rdev, sh->sector,
-+ STRIPE_SECTORS,
-+ &first_bad, &bad_sectors)) {
-+ set_bit(R5_MadeGood, &sh->dev[i].flags);
-+ if (test_bit(R5_ReadError, &sh->dev[i].flags))
-+ /* That was a successful write so make
-+ * sure it looks like we already did
-+ * a re-write.
-+ */
-+ set_bit(R5_ReWrite, &sh->dev[i].flags);
-+ }
-+ }
-+ rdev_dec_pending(rdev, conf->mddev);
-+
-+ if (sh->batch_head && !uptodate && !replacement)
-+ set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
-+
-+ if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
-+ clear_bit(R5_LOCKED, &sh->dev[i].flags);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ release_stripe(sh);
-+
-+ if (sh->batch_head && sh != sh->batch_head)
-+ release_stripe(sh->batch_head);
-+}
-+
-+static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
-+
-+static void raid5_build_block(struct stripe_head *sh, int i, int previous)
-+{
-+ struct r5dev *dev = &sh->dev[i];
-+
-+ bio_init(&dev->req);
-+ dev->req.bi_io_vec = &dev->vec;
-+ dev->req.bi_max_vecs = 1;
-+ dev->req.bi_private = sh;
-+
-+ bio_init(&dev->rreq);
-+ dev->rreq.bi_io_vec = &dev->rvec;
-+ dev->rreq.bi_max_vecs = 1;
-+ dev->rreq.bi_private = sh;
-+
-+ dev->flags = 0;
-+ dev->sector = compute_blocknr(sh, i, previous);
-+}
-+
-+static void error(struct mddev *mddev, struct md_rdev *rdev)
-+{
-+ char b[BDEVNAME_SIZE];
-+ struct r5conf *conf = mddev->private;
-+ unsigned long flags;
-+ pr_debug("raid456: error called\n");
-+
-+ spin_lock_irqsave(&conf->device_lock, flags);
-+ clear_bit(In_sync, &rdev->flags);
-+ mddev->degraded = calc_degraded(conf);
-+ spin_unlock_irqrestore(&conf->device_lock, flags);
-+ set_bit(MD_RECOVERY_INTR, &mddev->recovery);
-+
-+ set_bit(Blocked, &rdev->flags);
-+ set_bit(Faulty, &rdev->flags);
-+ set_bit(MD_CHANGE_DEVS, &mddev->flags);
-+ printk(KERN_ALERT
-+ "md/raid:%s: Disk failure on %s, disabling device.\n"
-+ "md/raid:%s: Operation continuing on %d devices.\n",
-+ mdname(mddev),
-+ bdevname(rdev->bdev, b),
-+ mdname(mddev),
-+ conf->raid_disks - mddev->degraded);
-+}
-+
-+/*
-+ * Input: a 'big' sector number,
-+ * Output: index of the data and parity disk, and the sector # in them.
-+ */
-+static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
-+ int previous, int *dd_idx,
-+ struct stripe_head *sh)
-+{
-+ sector_t stripe, stripe2;
-+ sector_t chunk_number;
-+ unsigned int chunk_offset;
-+ int pd_idx, qd_idx;
-+ int ddf_layout = 0;
-+ sector_t new_sector;
-+ int algorithm = previous ? conf->prev_algo
-+ : conf->algorithm;
-+ int sectors_per_chunk = previous ? conf->prev_chunk_sectors
-+ : conf->chunk_sectors;
-+ int raid_disks = previous ? conf->previous_raid_disks
-+ : conf->raid_disks;
-+ int data_disks = raid_disks - conf->max_degraded;
-+
-+ /* First compute the information on this sector */
-+
-+ /*
-+ * Compute the chunk number and the sector offset inside the chunk
-+ */
-+ chunk_offset = sector_div(r_sector, sectors_per_chunk);
-+ chunk_number = r_sector;
-+
-+ /*
-+ * Compute the stripe number
-+ */
-+ stripe = chunk_number;
-+ *dd_idx = sector_div(stripe, data_disks);
-+ stripe2 = stripe;
-+ /*
-+ * Select the parity disk based on the user selected algorithm.
-+ */
-+ pd_idx = qd_idx = -1;
-+ switch(conf->level) {
-+ case 4:
-+ pd_idx = data_disks;
-+ break;
-+ case 5:
-+ switch (algorithm) {
-+ case ALGORITHM_LEFT_ASYMMETRIC:
-+ pd_idx = data_disks - sector_div(stripe2, raid_disks);
-+ if (*dd_idx >= pd_idx)
-+ (*dd_idx)++;
-+ break;
-+ case ALGORITHM_RIGHT_ASYMMETRIC:
-+ pd_idx = sector_div(stripe2, raid_disks);
-+ if (*dd_idx >= pd_idx)
-+ (*dd_idx)++;
-+ break;
-+ case ALGORITHM_LEFT_SYMMETRIC:
-+ pd_idx = data_disks - sector_div(stripe2, raid_disks);
-+ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
-+ break;
-+ case ALGORITHM_RIGHT_SYMMETRIC:
-+ pd_idx = sector_div(stripe2, raid_disks);
-+ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
-+ break;
-+ case ALGORITHM_PARITY_0:
-+ pd_idx = 0;
-+ (*dd_idx)++;
-+ break;
-+ case ALGORITHM_PARITY_N:
-+ pd_idx = data_disks;
-+ break;
-+ default:
-+ BUG();
-+ }
-+ break;
-+ case 6:
-+
-+ switch (algorithm) {
-+ case ALGORITHM_LEFT_ASYMMETRIC:
-+ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
-+ qd_idx = pd_idx + 1;
-+ if (pd_idx == raid_disks-1) {
-+ (*dd_idx)++; /* Q D D D P */
-+ qd_idx = 0;
-+ } else if (*dd_idx >= pd_idx)
-+ (*dd_idx) += 2; /* D D P Q D */
-+ break;
-+ case ALGORITHM_RIGHT_ASYMMETRIC:
-+ pd_idx = sector_div(stripe2, raid_disks);
-+ qd_idx = pd_idx + 1;
-+ if (pd_idx == raid_disks-1) {
-+ (*dd_idx)++; /* Q D D D P */
-+ qd_idx = 0;
-+ } else if (*dd_idx >= pd_idx)
-+ (*dd_idx) += 2; /* D D P Q D */
-+ break;
-+ case ALGORITHM_LEFT_SYMMETRIC:
-+ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
-+ qd_idx = (pd_idx + 1) % raid_disks;
-+ *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
-+ break;
-+ case ALGORITHM_RIGHT_SYMMETRIC:
-+ pd_idx = sector_div(stripe2, raid_disks);
-+ qd_idx = (pd_idx + 1) % raid_disks;
-+ *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
-+ break;
-+
-+ case ALGORITHM_PARITY_0:
-+ pd_idx = 0;
-+ qd_idx = 1;
-+ (*dd_idx) += 2;
-+ break;
-+ case ALGORITHM_PARITY_N:
-+ pd_idx = data_disks;
-+ qd_idx = data_disks + 1;
-+ break;
-+
-+ case ALGORITHM_ROTATING_ZERO_RESTART:
-+ /* Exactly the same as RIGHT_ASYMMETRIC, but or
-+ * of blocks for computing Q is different.
-+ */
-+ pd_idx = sector_div(stripe2, raid_disks);
-+ qd_idx = pd_idx + 1;
-+ if (pd_idx == raid_disks-1) {
-+ (*dd_idx)++; /* Q D D D P */
-+ qd_idx = 0;
-+ } else if (*dd_idx >= pd_idx)
-+ (*dd_idx) += 2; /* D D P Q D */
-+ ddf_layout = 1;
-+ break;
-+
-+ case ALGORITHM_ROTATING_N_RESTART:
-+ /* Same a left_asymmetric, by first stripe is
-+ * D D D P Q rather than
-+ * Q D D D P
-+ */
-+ stripe2 += 1;
-+ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
-+ qd_idx = pd_idx + 1;
-+ if (pd_idx == raid_disks-1) {
-+ (*dd_idx)++; /* Q D D D P */
-+ qd_idx = 0;
-+ } else if (*dd_idx >= pd_idx)
-+ (*dd_idx) += 2; /* D D P Q D */
-+ ddf_layout = 1;
-+ break;
-+
-+ case ALGORITHM_ROTATING_N_CONTINUE:
-+ /* Same as left_symmetric but Q is before P */
-+ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
-+ qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
-+ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
-+ ddf_layout = 1;
-+ break;
-+
-+ case ALGORITHM_LEFT_ASYMMETRIC_6:
-+ /* RAID5 left_asymmetric, with Q on last device */
-+ pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
-+ if (*dd_idx >= pd_idx)
-+ (*dd_idx)++;
-+ qd_idx = raid_disks - 1;
-+ break;
-+
-+ case ALGORITHM_RIGHT_ASYMMETRIC_6:
-+ pd_idx = sector_div(stripe2, raid_disks-1);
-+ if (*dd_idx >= pd_idx)
-+ (*dd_idx)++;
-+ qd_idx = raid_disks - 1;
-+ break;
-+
-+ case ALGORITHM_LEFT_SYMMETRIC_6:
-+ pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
-+ *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
-+ qd_idx = raid_disks - 1;
-+ break;
-+
-+ case ALGORITHM_RIGHT_SYMMETRIC_6:
-+ pd_idx = sector_div(stripe2, raid_disks-1);
-+ *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
-+ qd_idx = raid_disks - 1;
-+ break;
-+
-+ case ALGORITHM_PARITY_0_6:
-+ pd_idx = 0;
-+ (*dd_idx)++;
-+ qd_idx = raid_disks - 1;
-+ break;
-+
-+ default:
-+ BUG();
-+ }
-+ break;
-+ }
-+
-+ if (sh) {
-+ sh->pd_idx = pd_idx;
-+ sh->qd_idx = qd_idx;
-+ sh->ddf_layout = ddf_layout;
-+ }
-+ /*
-+ * Finally, compute the new sector number
-+ */
-+ new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
-+ return new_sector;
-+}
-+
-+static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
-+{
-+ struct r5conf *conf = sh->raid_conf;
-+ int raid_disks = sh->disks;
-+ int data_disks = raid_disks - conf->max_degraded;
-+ sector_t new_sector = sh->sector, check;
-+ int sectors_per_chunk = previous ? conf->prev_chunk_sectors
-+ : conf->chunk_sectors;
-+ int algorithm = previous ? conf->prev_algo
-+ : conf->algorithm;
-+ sector_t stripe;
-+ int chunk_offset;
-+ sector_t chunk_number;
-+ int dummy1, dd_idx = i;
-+ sector_t r_sector;
-+ struct stripe_head sh2;
-+
-+ chunk_offset = sector_div(new_sector, sectors_per_chunk);
-+ stripe = new_sector;
-+
-+ if (i == sh->pd_idx)
-+ return 0;
-+ switch(conf->level) {
-+ case 4: break;
-+ case 5:
-+ switch (algorithm) {
-+ case ALGORITHM_LEFT_ASYMMETRIC:
-+ case ALGORITHM_RIGHT_ASYMMETRIC:
-+ if (i > sh->pd_idx)
-+ i--;
-+ break;
-+ case ALGORITHM_LEFT_SYMMETRIC:
-+ case ALGORITHM_RIGHT_SYMMETRIC:
-+ if (i < sh->pd_idx)
-+ i += raid_disks;
-+ i -= (sh->pd_idx + 1);
-+ break;
-+ case ALGORITHM_PARITY_0:
-+ i -= 1;
-+ break;
-+ case ALGORITHM_PARITY_N:
-+ break;
-+ default:
-+ BUG();
-+ }
-+ break;
-+ case 6:
-+ if (i == sh->qd_idx)
-+ return 0; /* It is the Q disk */
-+ switch (algorithm) {
-+ case ALGORITHM_LEFT_ASYMMETRIC:
-+ case ALGORITHM_RIGHT_ASYMMETRIC:
-+ case ALGORITHM_ROTATING_ZERO_RESTART:
-+ case ALGORITHM_ROTATING_N_RESTART:
-+ if (sh->pd_idx == raid_disks-1)
-+ i--; /* Q D D D P */
-+ else if (i > sh->pd_idx)
-+ i -= 2; /* D D P Q D */
-+ break;
-+ case ALGORITHM_LEFT_SYMMETRIC:
-+ case ALGORITHM_RIGHT_SYMMETRIC:
-+ if (sh->pd_idx == raid_disks-1)
-+ i--; /* Q D D D P */
-+ else {
-+ /* D D P Q D */
-+ if (i < sh->pd_idx)
-+ i += raid_disks;
-+ i -= (sh->pd_idx + 2);
-+ }
-+ break;
-+ case ALGORITHM_PARITY_0:
-+ i -= 2;
-+ break;
-+ case ALGORITHM_PARITY_N:
-+ break;
-+ case ALGORITHM_ROTATING_N_CONTINUE:
-+ /* Like left_symmetric, but P is before Q */
-+ if (sh->pd_idx == 0)
-+ i--; /* P D D D Q */
-+ else {
-+ /* D D Q P D */
-+ if (i < sh->pd_idx)
-+ i += raid_disks;
-+ i -= (sh->pd_idx + 1);
-+ }
-+ break;
-+ case ALGORITHM_LEFT_ASYMMETRIC_6:
-+ case ALGORITHM_RIGHT_ASYMMETRIC_6:
-+ if (i > sh->pd_idx)
-+ i--;
-+ break;
-+ case ALGORITHM_LEFT_SYMMETRIC_6:
-+ case ALGORITHM_RIGHT_SYMMETRIC_6:
-+ if (i < sh->pd_idx)
-+ i += data_disks + 1;
-+ i -= (sh->pd_idx + 1);
-+ break;
-+ case ALGORITHM_PARITY_0_6:
-+ i -= 1;
-+ break;
-+ default:
-+ BUG();
-+ }
-+ break;
-+ }
-+
-+ chunk_number = stripe * data_disks + i;
-+ r_sector = chunk_number * sectors_per_chunk + chunk_offset;
-+
-+ check = raid5_compute_sector(conf, r_sector,
-+ previous, &dummy1, &sh2);
-+ if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
-+ || sh2.qd_idx != sh->qd_idx) {
-+ printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
-+ mdname(conf->mddev));
-+ return 0;
-+ }
-+ return r_sector;
-+}
-+
-+static void
-+schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
-+ int rcw, int expand)
-+{
-+ int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
-+ struct r5conf *conf = sh->raid_conf;
-+ int level = conf->level;
-+
-+ if (rcw) {
-+
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+
-+ if (dev->towrite) {
-+ set_bit(R5_LOCKED, &dev->flags);
-+ set_bit(R5_Wantdrain, &dev->flags);
-+ if (!expand)
-+ clear_bit(R5_UPTODATE, &dev->flags);
-+ s->locked++;
-+ }
-+ }
-+ /* if we are not expanding this is a proper write request, and
-+ * there will be bios with new data to be drained into the
-+ * stripe cache
-+ */
-+ if (!expand) {
-+ if (!s->locked)
-+ /* False alarm, nothing to do */
-+ return;
-+ sh->reconstruct_state = reconstruct_state_drain_run;
-+ set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
-+ } else
-+ sh->reconstruct_state = reconstruct_state_run;
-+
-+ set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
-+
-+ if (s->locked + conf->max_degraded == disks)
-+ if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
-+ atomic_inc(&conf->pending_full_writes);
-+ } else {
-+ BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
-+ test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
-+ BUG_ON(level == 6 &&
-+ (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
-+ test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
-+
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+ if (i == pd_idx || i == qd_idx)
-+ continue;
-+
-+ if (dev->towrite &&
-+ (test_bit(R5_UPTODATE, &dev->flags) ||
-+ test_bit(R5_Wantcompute, &dev->flags))) {
-+ set_bit(R5_Wantdrain, &dev->flags);
-+ set_bit(R5_LOCKED, &dev->flags);
-+ clear_bit(R5_UPTODATE, &dev->flags);
-+ s->locked++;
-+ }
-+ }
-+ if (!s->locked)
-+ /* False alarm - nothing to do */
-+ return;
-+ sh->reconstruct_state = reconstruct_state_prexor_drain_run;
-+ set_bit(STRIPE_OP_PREXOR, &s->ops_request);
-+ set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
-+ set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
-+ }
-+
-+ /* keep the parity disk(s) locked while asynchronous operations
-+ * are in flight
-+ */
-+ set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
-+ clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
-+ s->locked++;
-+
-+ if (level == 6) {
-+ int qd_idx = sh->qd_idx;
-+ struct r5dev *dev = &sh->dev[qd_idx];
-+
-+ set_bit(R5_LOCKED, &dev->flags);
-+ clear_bit(R5_UPTODATE, &dev->flags);
-+ s->locked++;
-+ }
-+
-+ pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
-+ __func__, (unsigned long long)sh->sector,
-+ s->locked, s->ops_request);
-+}
-+
-+/*
-+ * Each stripe/dev can have one or more bion attached.
-+ * toread/towrite point to the first in a chain.
-+ * The bi_next chain must be in order.
-+ */
-+static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,
-+ int forwrite, int previous)
-+{
-+ struct bio **bip;
-+ struct r5conf *conf = sh->raid_conf;
-+ int firstwrite=0;
-+
-+ pr_debug("adding bi b#%llu to stripe s#%llu\n",
-+ (unsigned long long)bi->bi_iter.bi_sector,
-+ (unsigned long long)sh->sector);
-+
-+ /*
-+ * If several bio share a stripe. The bio bi_phys_segments acts as a
-+ * reference count to avoid race. The reference count should already be
-+ * increased before this function is called (for example, in
-+ * make_request()), so other bio sharing this stripe will not free the
-+ * stripe. If a stripe is owned by one stripe, the stripe lock will
-+ * protect it.
-+ */
-+ spin_lock_irq(&sh->stripe_lock);
-+ /* Don't allow new IO added to stripes in batch list */
-+ if (sh->batch_head)
-+ goto overlap;
-+ if (forwrite) {
-+ bip = &sh->dev[dd_idx].towrite;
-+ if (*bip == NULL)
-+ firstwrite = 1;
-+ } else
-+ bip = &sh->dev[dd_idx].toread;
-+ while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
-+ if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
-+ goto overlap;
-+ bip = & (*bip)->bi_next;
-+ }
-+ if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
-+ goto overlap;
-+
-+ if (!forwrite || previous)
-+ clear_bit(STRIPE_BATCH_READY, &sh->state);
-+
-+ BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
-+ if (*bip)
-+ bi->bi_next = *bip;
-+ *bip = bi;
-+ raid5_inc_bi_active_stripes(bi);
-+
-+ if (forwrite) {
-+ /* check if page is covered */
-+ sector_t sector = sh->dev[dd_idx].sector;
-+ for (bi=sh->dev[dd_idx].towrite;
-+ sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
-+ bi && bi->bi_iter.bi_sector <= sector;
-+ bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
-+ if (bio_end_sector(bi) >= sector)
-+ sector = bio_end_sector(bi);
-+ }
-+ if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
-+ if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
-+ sh->overwrite_disks++;
-+ }
-+
-+ pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
-+ (unsigned long long)(*bip)->bi_iter.bi_sector,
-+ (unsigned long long)sh->sector, dd_idx);
-+
-+ if (conf->mddev->bitmap && firstwrite) {
-+ /* Cannot hold spinlock over bitmap_startwrite,
-+ * but must ensure this isn't added to a batch until
-+ * we have added to the bitmap and set bm_seq.
-+ * So set STRIPE_BITMAP_PENDING to prevent
-+ * batching.
-+ * If multiple add_stripe_bio() calls race here they
-+ * much all set STRIPE_BITMAP_PENDING. So only the first one
-+ * to complete "bitmap_startwrite" gets to set
-+ * STRIPE_BIT_DELAY. This is important as once a stripe
-+ * is added to a batch, STRIPE_BIT_DELAY cannot be changed
-+ * any more.
-+ */
-+ set_bit(STRIPE_BITMAP_PENDING, &sh->state);
-+ spin_unlock_irq(&sh->stripe_lock);
-+ bitmap_startwrite(conf->mddev->bitmap, sh->sector,
-+ STRIPE_SECTORS, 0);
-+ spin_lock_irq(&sh->stripe_lock);
-+ clear_bit(STRIPE_BITMAP_PENDING, &sh->state);
-+ if (!sh->batch_head) {
-+ sh->bm_seq = conf->seq_flush+1;
-+ set_bit(STRIPE_BIT_DELAY, &sh->state);
-+ }
-+ }
-+ spin_unlock_irq(&sh->stripe_lock);
-+
-+ if (stripe_can_batch(sh))
-+ stripe_add_to_batch_list(conf, sh);
-+ return 1;
-+
-+ overlap:
-+ set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
-+ spin_unlock_irq(&sh->stripe_lock);
-+ return 0;
-+}
-+
-+static void end_reshape(struct r5conf *conf);
-+
-+static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
-+ struct stripe_head *sh)
-+{
-+ int sectors_per_chunk =
-+ previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
-+ int dd_idx;
-+ int chunk_offset = sector_div(stripe, sectors_per_chunk);
-+ int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
-+
-+ raid5_compute_sector(conf,
-+ stripe * (disks - conf->max_degraded)
-+ *sectors_per_chunk + chunk_offset,
-+ previous,
-+ &dd_idx, sh);
-+}
-+
-+static void
-+handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
-+ struct stripe_head_state *s, int disks,
-+ struct bio **return_bi)
-+{
-+ int i;
-+ BUG_ON(sh->batch_head);
-+ for (i = disks; i--; ) {
-+ struct bio *bi;
-+ int bitmap_end = 0;
-+
-+ if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
-+ struct md_rdev *rdev;
-+ rcu_read_lock();
-+ rdev = rcu_dereference(conf->disks[i].rdev);
-+ if (rdev && test_bit(In_sync, &rdev->flags))
-+ atomic_inc(&rdev->nr_pending);
-+ else
-+ rdev = NULL;
-+ rcu_read_unlock();
-+ if (rdev) {
-+ if (!rdev_set_badblocks(
-+ rdev,
-+ sh->sector,
-+ STRIPE_SECTORS, 0))
-+ md_error(conf->mddev, rdev);
-+ rdev_dec_pending(rdev, conf->mddev);
-+ }
-+ }
-+ spin_lock_irq(&sh->stripe_lock);
-+ /* fail all writes first */
-+ bi = sh->dev[i].towrite;
-+ sh->dev[i].towrite = NULL;
-+ sh->overwrite_disks = 0;
-+ spin_unlock_irq(&sh->stripe_lock);
-+ if (bi)
-+ bitmap_end = 1;
-+
-+ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
-+ wake_up(&conf->wait_for_overlap);
-+
-+ while (bi && bi->bi_iter.bi_sector <
-+ sh->dev[i].sector + STRIPE_SECTORS) {
-+ struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
-+ clear_bit(BIO_UPTODATE, &bi->bi_flags);
-+ if (!raid5_dec_bi_active_stripes(bi)) {
-+ md_write_end(conf->mddev);
-+ bi->bi_next = *return_bi;
-+ *return_bi = bi;
-+ }
-+ bi = nextbi;
-+ }
-+ if (bitmap_end)
-+ bitmap_endwrite(conf->mddev->bitmap, sh->sector,
-+ STRIPE_SECTORS, 0, 0);
-+ bitmap_end = 0;
-+ /* and fail all 'written' */
-+ bi = sh->dev[i].written;
-+ sh->dev[i].written = NULL;
-+ if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
-+ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
-+ sh->dev[i].page = sh->dev[i].orig_page;
-+ }
-+
-+ if (bi) bitmap_end = 1;
-+ while (bi && bi->bi_iter.bi_sector <
-+ sh->dev[i].sector + STRIPE_SECTORS) {
-+ struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
-+ clear_bit(BIO_UPTODATE, &bi->bi_flags);
-+ if (!raid5_dec_bi_active_stripes(bi)) {
-+ md_write_end(conf->mddev);
-+ bi->bi_next = *return_bi;
-+ *return_bi = bi;
-+ }
-+ bi = bi2;
-+ }
-+
-+ /* fail any reads if this device is non-operational and
-+ * the data has not reached the cache yet.
-+ */
-+ if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
-+ (!test_bit(R5_Insync, &sh->dev[i].flags) ||
-+ test_bit(R5_ReadError, &sh->dev[i].flags))) {
-+ spin_lock_irq(&sh->stripe_lock);
-+ bi = sh->dev[i].toread;
-+ sh->dev[i].toread = NULL;
-+ spin_unlock_irq(&sh->stripe_lock);
-+ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
-+ wake_up(&conf->wait_for_overlap);
-+ while (bi && bi->bi_iter.bi_sector <
-+ sh->dev[i].sector + STRIPE_SECTORS) {
-+ struct bio *nextbi =
-+ r5_next_bio(bi, sh->dev[i].sector);
-+ clear_bit(BIO_UPTODATE, &bi->bi_flags);
-+ if (!raid5_dec_bi_active_stripes(bi)) {
-+ bi->bi_next = *return_bi;
-+ *return_bi = bi;
-+ }
-+ bi = nextbi;
-+ }
-+ }
-+ if (bitmap_end)
-+ bitmap_endwrite(conf->mddev->bitmap, sh->sector,
-+ STRIPE_SECTORS, 0, 0);
-+ /* If we were in the middle of a write the parity block might
-+ * still be locked - so just clear all R5_LOCKED flags
-+ */
-+ clear_bit(R5_LOCKED, &sh->dev[i].flags);
-+ }
-+
-+ if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
-+ if (atomic_dec_and_test(&conf->pending_full_writes))
-+ md_wakeup_thread(conf->mddev->thread);
-+}
-+
-+static void
-+handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
-+ struct stripe_head_state *s)
-+{
-+ int abort = 0;
-+ int i;
-+
-+ BUG_ON(sh->batch_head);
-+ clear_bit(STRIPE_SYNCING, &sh->state);
-+ if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
-+ wake_up(&conf->wait_for_overlap);
-+ s->syncing = 0;
-+ s->replacing = 0;
-+ /* There is nothing more to do for sync/check/repair.
-+ * Don't even need to abort as that is handled elsewhere
-+ * if needed, and not always wanted e.g. if there is a known
-+ * bad block here.
-+ * For recover/replace we need to record a bad block on all
-+ * non-sync devices, or abort the recovery
-+ */
-+ if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
-+ /* During recovery devices cannot be removed, so
-+ * locking and refcounting of rdevs is not needed
-+ */
-+ for (i = 0; i < conf->raid_disks; i++) {
-+ struct md_rdev *rdev = conf->disks[i].rdev;
-+ if (rdev
-+ && !test_bit(Faulty, &rdev->flags)
-+ && !test_bit(In_sync, &rdev->flags)
-+ && !rdev_set_badblocks(rdev, sh->sector,
-+ STRIPE_SECTORS, 0))
-+ abort = 1;
-+ rdev = conf->disks[i].replacement;
-+ if (rdev
-+ && !test_bit(Faulty, &rdev->flags)
-+ && !test_bit(In_sync, &rdev->flags)
-+ && !rdev_set_badblocks(rdev, sh->sector,
-+ STRIPE_SECTORS, 0))
-+ abort = 1;
-+ }
-+ if (abort)
-+ conf->recovery_disabled =
-+ conf->mddev->recovery_disabled;
-+ }
-+ md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
-+}
-+
-+static int want_replace(struct stripe_head *sh, int disk_idx)
-+{
-+ struct md_rdev *rdev;
-+ int rv = 0;
-+ /* Doing recovery so rcu locking not required */
-+ rdev = sh->raid_conf->disks[disk_idx].replacement;
-+ if (rdev
-+ && !test_bit(Faulty, &rdev->flags)
-+ && !test_bit(In_sync, &rdev->flags)
-+ && (rdev->recovery_offset <= sh->sector
-+ || rdev->mddev->recovery_cp <= sh->sector))
-+ rv = 1;
-+
-+ return rv;
-+}
-+
-+/* fetch_block - checks the given member device to see if its data needs
-+ * to be read or computed to satisfy a request.
-+ *
-+ * Returns 1 when no more member devices need to be checked, otherwise returns
-+ * 0 to tell the loop in handle_stripe_fill to continue
-+ */
-+
-+static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
-+ int disk_idx, int disks)
-+{
-+ struct r5dev *dev = &sh->dev[disk_idx];
-+ struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
-+ &sh->dev[s->failed_num[1]] };
-+ int i;
-+
-+
-+ if (test_bit(R5_LOCKED, &dev->flags) ||
-+ test_bit(R5_UPTODATE, &dev->flags))
-+ /* No point reading this as we already have it or have
-+ * decided to get it.
-+ */
-+ return 0;
-+
-+ if (dev->toread ||
-+ (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)))
-+ /* We need this block to directly satisfy a request */
-+ return 1;
-+
-+ if (s->syncing || s->expanding ||
-+ (s->replacing && want_replace(sh, disk_idx)))
-+ /* When syncing, or expanding we read everything.
-+ * When replacing, we need the replaced block.
-+ */
-+ return 1;
-+
-+ if ((s->failed >= 1 && fdev[0]->toread) ||
-+ (s->failed >= 2 && fdev[1]->toread))
-+ /* If we want to read from a failed device, then
-+ * we need to actually read every other device.
-+ */
-+ return 1;
-+
-+ /* Sometimes neither read-modify-write nor reconstruct-write
-+ * cycles can work. In those cases we read every block we
-+ * can. Then the parity-update is certain to have enough to
-+ * work with.
-+ * This can only be a problem when we need to write something,
-+ * and some device has failed. If either of those tests
-+ * fail we need look no further.
-+ */
-+ if (!s->failed || !s->to_write)
-+ return 0;
-+
-+ if (test_bit(R5_Insync, &dev->flags) &&
-+ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-+ /* Pre-reads at not permitted until after short delay
-+ * to gather multiple requests. However if this
-+ * device is no Insync, the block could only be be computed
-+ * and there is no need to delay that.
-+ */
-+ return 0;
-+
-+ for (i = 0; i < s->failed; i++) {
-+ if (fdev[i]->towrite &&
-+ !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
-+ !test_bit(R5_OVERWRITE, &fdev[i]->flags))
-+ /* If we have a partial write to a failed
-+ * device, then we will need to reconstruct
-+ * the content of that device, so all other
-+ * devices must be read.
-+ */
-+ return 1;
-+ }
-+
-+ /* If we are forced to do a reconstruct-write, either because
-+ * the current RAID6 implementation only supports that, or
-+ * or because parity cannot be trusted and we are currently
-+ * recovering it, there is extra need to be careful.
-+ * If one of the devices that we would need to read, because
-+ * it is not being overwritten (and maybe not written at all)
-+ * is missing/faulty, then we need to read everything we can.
-+ */
-+ if (sh->raid_conf->level != 6 &&
-+ sh->sector < sh->raid_conf->mddev->recovery_cp)
-+ /* reconstruct-write isn't being forced */
-+ return 0;
-+ for (i = 0; i < s->failed; i++) {
-+ if (s->failed_num[i] != sh->pd_idx &&
-+ s->failed_num[i] != sh->qd_idx &&
-+ !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
-+ !test_bit(R5_OVERWRITE, &fdev[i]->flags))
-+ return 1;
-+ }
-+
-+ return 0;
-+}
-+
-+static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
-+ int disk_idx, int disks)
-+{
-+ struct r5dev *dev = &sh->dev[disk_idx];
-+
-+ /* is the data in this block needed, and can we get it? */
-+ if (need_this_block(sh, s, disk_idx, disks)) {
-+ /* we would like to get this block, possibly by computing it,
-+ * otherwise read it if the backing disk is insync
-+ */
-+ BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
-+ BUG_ON(test_bit(R5_Wantread, &dev->flags));
-+ BUG_ON(sh->batch_head);
-+ if ((s->uptodate == disks - 1) &&
-+ (s->failed && (disk_idx == s->failed_num[0] ||
-+ disk_idx == s->failed_num[1]))) {
-+ /* have disk failed, and we're requested to fetch it;
-+ * do compute it
-+ */
-+ pr_debug("Computing stripe %llu block %d\n",
-+ (unsigned long long)sh->sector, disk_idx);
-+ set_bit(STRIPE_COMPUTE_RUN, &sh->state);
-+ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
-+ set_bit(R5_Wantcompute, &dev->flags);
-+ sh->ops.target = disk_idx;
-+ sh->ops.target2 = -1; /* no 2nd target */
-+ s->req_compute = 1;
-+ /* Careful: from this point on 'uptodate' is in the eye
-+ * of raid_run_ops which services 'compute' operations
-+ * before writes. R5_Wantcompute flags a block that will
-+ * be R5_UPTODATE by the time it is needed for a
-+ * subsequent operation.
-+ */
-+ s->uptodate++;
-+ return 1;
-+ } else if (s->uptodate == disks-2 && s->failed >= 2) {
-+ /* Computing 2-failure is *very* expensive; only
-+ * do it if failed >= 2
-+ */
-+ int other;
-+ for (other = disks; other--; ) {
-+ if (other == disk_idx)
-+ continue;
-+ if (!test_bit(R5_UPTODATE,
-+ &sh->dev[other].flags))
-+ break;
-+ }
-+ BUG_ON(other < 0);
-+ pr_debug("Computing stripe %llu blocks %d,%d\n",
-+ (unsigned long long)sh->sector,
-+ disk_idx, other);
-+ set_bit(STRIPE_COMPUTE_RUN, &sh->state);
-+ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
-+ set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
-+ set_bit(R5_Wantcompute, &sh->dev[other].flags);
-+ sh->ops.target = disk_idx;
-+ sh->ops.target2 = other;
-+ s->uptodate += 2;
-+ s->req_compute = 1;
-+ return 1;
-+ } else if (test_bit(R5_Insync, &dev->flags)) {
-+ set_bit(R5_LOCKED, &dev->flags);
-+ set_bit(R5_Wantread, &dev->flags);
-+ s->locked++;
-+ pr_debug("Reading block %d (sync=%d)\n",
-+ disk_idx, s->syncing);
-+ }
-+ }
-+
-+ return 0;
-+}
-+
-+/**
-+ * handle_stripe_fill - read or compute data to satisfy pending requests.
-+ */
-+static void handle_stripe_fill(struct stripe_head *sh,
-+ struct stripe_head_state *s,
-+ int disks)
-+{
-+ int i;
-+
-+ /* look for blocks to read/compute, skip this if a compute
-+ * is already in flight, or if the stripe contents are in the
-+ * midst of changing due to a write
-+ */
-+ if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
-+ !sh->reconstruct_state)
-+ for (i = disks; i--; )
-+ if (fetch_block(sh, s, i, disks))
-+ break;
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+}
-+
-+static void break_stripe_batch_list(struct stripe_head *head_sh,
-+ unsigned long handle_flags);
-+/* handle_stripe_clean_event
-+ * any written block on an uptodate or failed drive can be returned.
-+ * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
-+ * never LOCKED, so we don't need to test 'failed' directly.
-+ */
-+static void handle_stripe_clean_event(struct r5conf *conf,
-+ struct stripe_head *sh, int disks, struct bio **return_bi)
-+{
-+ int i;
-+ struct r5dev *dev;
-+ int discard_pending = 0;
-+ struct stripe_head *head_sh = sh;
-+ bool do_endio = false;
-+
-+ for (i = disks; i--; )
-+ if (sh->dev[i].written) {
-+ dev = &sh->dev[i];
-+ if (!test_bit(R5_LOCKED, &dev->flags) &&
-+ (test_bit(R5_UPTODATE, &dev->flags) ||
-+ test_bit(R5_Discard, &dev->flags) ||
-+ test_bit(R5_SkipCopy, &dev->flags))) {
-+ /* We can return any write requests */
-+ struct bio *wbi, *wbi2;
-+ pr_debug("Return write for disc %d\n", i);
-+ if (test_and_clear_bit(R5_Discard, &dev->flags))
-+ clear_bit(R5_UPTODATE, &dev->flags);
-+ if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {
-+ WARN_ON(test_bit(R5_UPTODATE, &dev->flags));
-+ }
-+ do_endio = true;
-+
-+returnbi:
-+ dev->page = dev->orig_page;
-+ wbi = dev->written;
-+ dev->written = NULL;
-+ while (wbi && wbi->bi_iter.bi_sector <
-+ dev->sector + STRIPE_SECTORS) {
-+ wbi2 = r5_next_bio(wbi, dev->sector);
-+ if (!raid5_dec_bi_active_stripes(wbi)) {
-+ md_write_end(conf->mddev);
-+ wbi->bi_next = *return_bi;
-+ *return_bi = wbi;
-+ }
-+ wbi = wbi2;
-+ }
-+ bitmap_endwrite(conf->mddev->bitmap, sh->sector,
-+ STRIPE_SECTORS,
-+ !test_bit(STRIPE_DEGRADED, &sh->state),
-+ 0);
-+ if (head_sh->batch_head) {
-+ sh = list_first_entry(&sh->batch_list,
-+ struct stripe_head,
-+ batch_list);
-+ if (sh != head_sh) {
-+ dev = &sh->dev[i];
-+ goto returnbi;
-+ }
-+ }
-+ sh = head_sh;
-+ dev = &sh->dev[i];
-+ } else if (test_bit(R5_Discard, &dev->flags))
-+ discard_pending = 1;
-+ WARN_ON(test_bit(R5_SkipCopy, &dev->flags));
-+ WARN_ON(dev->page != dev->orig_page);
-+ }
-+ if (!discard_pending &&
-+ test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
-+ clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
-+ clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
-+ if (sh->qd_idx >= 0) {
-+ clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
-+ clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
-+ }
-+ /* now that discard is done we can proceed with any sync */
-+ clear_bit(STRIPE_DISCARD, &sh->state);
-+ /*
-+ * SCSI discard will change some bio fields and the stripe has
-+ * no updated data, so remove it from hash list and the stripe
-+ * will be reinitialized
-+ */
-+ spin_lock_irq(&conf->device_lock);
-+unhash:
-+ remove_hash(sh);
-+ if (head_sh->batch_head) {
-+ sh = list_first_entry(&sh->batch_list,
-+ struct stripe_head, batch_list);
-+ if (sh != head_sh)
-+ goto unhash;
-+ }
-+ spin_unlock_irq(&conf->device_lock);
-+ sh = head_sh;
-+
-+ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+
-+ }
-+
-+ if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
-+ if (atomic_dec_and_test(&conf->pending_full_writes))
-+ md_wakeup_thread(conf->mddev->thread);
-+
-+ if (head_sh->batch_head && do_endio)
-+ break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS);
-+}
-+
-+static void handle_stripe_dirtying(struct r5conf *conf,
-+ struct stripe_head *sh,
-+ struct stripe_head_state *s,
-+ int disks)
-+{
-+ int rmw = 0, rcw = 0, i;
-+ sector_t recovery_cp = conf->mddev->recovery_cp;
-+
-+ /* Check whether resync is now happening or should start.
-+ * If yes, then the array is dirty (after unclean shutdown or
-+ * initial creation), so parity in some stripes might be inconsistent.
-+ * In this case, we need to always do reconstruct-write, to ensure
-+ * that in case of drive failure or read-error correction, we
-+ * generate correct data from the parity.
-+ */
-+ if (conf->rmw_level == PARITY_DISABLE_RMW ||
-+ (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
-+ s->failed == 0)) {
-+ /* Calculate the real rcw later - for now make it
-+ * look like rcw is cheaper
-+ */
-+ rcw = 1; rmw = 2;
-+ pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",
-+ conf->rmw_level, (unsigned long long)recovery_cp,
-+ (unsigned long long)sh->sector);
-+ } else for (i = disks; i--; ) {
-+ /* would I have to read this buffer for read_modify_write */
-+ struct r5dev *dev = &sh->dev[i];
-+ if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
-+ !test_bit(R5_LOCKED, &dev->flags) &&
-+ !(test_bit(R5_UPTODATE, &dev->flags) ||
-+ test_bit(R5_Wantcompute, &dev->flags))) {
-+ if (test_bit(R5_Insync, &dev->flags))
-+ rmw++;
-+ else
-+ rmw += 2*disks; /* cannot read it */
-+ }
-+ /* Would I have to read this buffer for reconstruct_write */
-+ if (!test_bit(R5_OVERWRITE, &dev->flags) &&
-+ i != sh->pd_idx && i != sh->qd_idx &&
-+ !test_bit(R5_LOCKED, &dev->flags) &&
-+ !(test_bit(R5_UPTODATE, &dev->flags) ||
-+ test_bit(R5_Wantcompute, &dev->flags))) {
-+ if (test_bit(R5_Insync, &dev->flags))
-+ rcw++;
-+ else
-+ rcw += 2*disks;
-+ }
-+ }
-+ pr_debug("for sector %llu, rmw=%d rcw=%d\n",
-+ (unsigned long long)sh->sector, rmw, rcw);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_ENABLE_RMW)) && rmw > 0) {
-+ /* prefer read-modify-write, but need to get some data */
-+ if (conf->mddev->queue)
-+ blk_add_trace_msg(conf->mddev->queue,
-+ "raid5 rmw %llu %d",
-+ (unsigned long long)sh->sector, rmw);
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+ if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
-+ !test_bit(R5_LOCKED, &dev->flags) &&
-+ !(test_bit(R5_UPTODATE, &dev->flags) ||
-+ test_bit(R5_Wantcompute, &dev->flags)) &&
-+ test_bit(R5_Insync, &dev->flags)) {
-+ if (test_bit(STRIPE_PREREAD_ACTIVE,
-+ &sh->state)) {
-+ pr_debug("Read_old block %d for r-m-w\n",
-+ i);
-+ set_bit(R5_LOCKED, &dev->flags);
-+ set_bit(R5_Wantread, &dev->flags);
-+ s->locked++;
-+ } else {
-+ set_bit(STRIPE_DELAYED, &sh->state);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ }
-+ }
-+ }
-+ }
-+ if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_ENABLE_RMW)) && rcw > 0) {
-+ /* want reconstruct write, but need to get some data */
-+ int qread =0;
-+ rcw = 0;
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+ if (!test_bit(R5_OVERWRITE, &dev->flags) &&
-+ i != sh->pd_idx && i != sh->qd_idx &&
-+ !test_bit(R5_LOCKED, &dev->flags) &&
-+ !(test_bit(R5_UPTODATE, &dev->flags) ||
-+ test_bit(R5_Wantcompute, &dev->flags))) {
-+ rcw++;
-+ if (test_bit(R5_Insync, &dev->flags) &&
-+ test_bit(STRIPE_PREREAD_ACTIVE,
-+ &sh->state)) {
-+ pr_debug("Read_old block "
-+ "%d for Reconstruct\n", i);
-+ set_bit(R5_LOCKED, &dev->flags);
-+ set_bit(R5_Wantread, &dev->flags);
-+ s->locked++;
-+ qread++;
-+ } else {
-+ set_bit(STRIPE_DELAYED, &sh->state);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ }
-+ }
-+ }
-+ if (rcw && conf->mddev->queue)
-+ blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
-+ (unsigned long long)sh->sector,
-+ rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
-+ }
-+
-+ if (rcw > disks && rmw > disks &&
-+ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-+ set_bit(STRIPE_DELAYED, &sh->state);
-+
-+ /* now if nothing is locked, and if we have enough data,
-+ * we can start a write request
-+ */
-+ /* since handle_stripe can be called at any time we need to handle the
-+ * case where a compute block operation has been submitted and then a
-+ * subsequent call wants to start a write request. raid_run_ops only
-+ * handles the case where compute block and reconstruct are requested
-+ * simultaneously. If this is not the case then new writes need to be
-+ * held off until the compute completes.
-+ */
-+ if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
-+ (s->locked == 0 && (rcw == 0 || rmw == 0) &&
-+ !test_bit(STRIPE_BIT_DELAY, &sh->state)))
-+ schedule_reconstruction(sh, s, rcw == 0, 0);
-+}
-+
-+static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
-+ struct stripe_head_state *s, int disks)
-+{
-+ struct r5dev *dev = NULL;
-+
-+ BUG_ON(sh->batch_head);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+
-+ switch (sh->check_state) {
-+ case check_state_idle:
-+ /* start a new check operation if there are no failures */
-+ if (s->failed == 0) {
-+ BUG_ON(s->uptodate != disks);
-+ sh->check_state = check_state_run;
-+ set_bit(STRIPE_OP_CHECK, &s->ops_request);
-+ clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
-+ s->uptodate--;
-+ break;
-+ }
-+ dev = &sh->dev[s->failed_num[0]];
-+ /* fall through */
-+ case check_state_compute_result:
-+ sh->check_state = check_state_idle;
-+ if (!dev)
-+ dev = &sh->dev[sh->pd_idx];
-+
-+ /* check that a write has not made the stripe insync */
-+ if (test_bit(STRIPE_INSYNC, &sh->state))
-+ break;
-+
-+ /* either failed parity check, or recovery is happening */
-+ BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
-+ BUG_ON(s->uptodate != disks);
-+
-+ set_bit(R5_LOCKED, &dev->flags);
-+ s->locked++;
-+ set_bit(R5_Wantwrite, &dev->flags);
-+
-+ clear_bit(STRIPE_DEGRADED, &sh->state);
-+ set_bit(STRIPE_INSYNC, &sh->state);
-+ break;
-+ case check_state_run:
-+ break; /* we will be called again upon completion */
-+ case check_state_check_result:
-+ sh->check_state = check_state_idle;
-+
-+ /* if a failure occurred during the check operation, leave
-+ * STRIPE_INSYNC not set and let the stripe be handled again
-+ */
-+ if (s->failed)
-+ break;
-+
-+ /* handle a successful check operation, if parity is correct
-+ * we are done. Otherwise update the mismatch count and repair
-+ * parity if !MD_RECOVERY_CHECK
-+ */
-+ if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
-+ /* parity is correct (on disc,
-+ * not in buffer any more)
-+ */
-+ set_bit(STRIPE_INSYNC, &sh->state);
-+ else {
-+ atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
-+ if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
-+ /* don't try to repair!! */
-+ set_bit(STRIPE_INSYNC, &sh->state);
-+ else {
-+ sh->check_state = check_state_compute_run;
-+ set_bit(STRIPE_COMPUTE_RUN, &sh->state);
-+ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
-+ set_bit(R5_Wantcompute,
-+ &sh->dev[sh->pd_idx].flags);
-+ sh->ops.target = sh->pd_idx;
-+ sh->ops.target2 = -1;
-+ s->uptodate++;
-+ }
-+ }
-+ break;
-+ case check_state_compute_run:
-+ break;
-+ default:
-+ printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
-+ __func__, sh->check_state,
-+ (unsigned long long) sh->sector);
-+ BUG();
-+ }
-+}
-+
-+static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
-+ struct stripe_head_state *s,
-+ int disks)
-+{
-+ int pd_idx = sh->pd_idx;
-+ int qd_idx = sh->qd_idx;
-+ struct r5dev *dev;
-+
-+ BUG_ON(sh->batch_head);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+
-+ BUG_ON(s->failed > 2);
-+
-+ /* Want to check and possibly repair P and Q.
-+ * However there could be one 'failed' device, in which
-+ * case we can only check one of them, possibly using the
-+ * other to generate missing data
-+ */
-+
-+ switch (sh->check_state) {
-+ case check_state_idle:
-+ /* start a new check operation if there are < 2 failures */
-+ if (s->failed == s->q_failed) {
-+ /* The only possible failed device holds Q, so it
-+ * makes sense to check P (If anything else were failed,
-+ * we would have used P to recreate it).
-+ */
-+ sh->check_state = check_state_run;
-+ }
-+ if (!s->q_failed && s->failed < 2) {
-+ /* Q is not failed, and we didn't use it to generate
-+ * anything, so it makes sense to check it
-+ */
-+ if (sh->check_state == check_state_run)
-+ sh->check_state = check_state_run_pq;
-+ else
-+ sh->check_state = check_state_run_q;
-+ }
-+
-+ /* discard potentially stale zero_sum_result */
-+ sh->ops.zero_sum_result = 0;
-+
-+ if (sh->check_state == check_state_run) {
-+ /* async_xor_zero_sum destroys the contents of P */
-+ clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
-+ s->uptodate--;
-+ }
-+ if (sh->check_state >= check_state_run &&
-+ sh->check_state <= check_state_run_pq) {
-+ /* async_syndrome_zero_sum preserves P and Q, so
-+ * no need to mark them !uptodate here
-+ */
-+ set_bit(STRIPE_OP_CHECK, &s->ops_request);
-+ break;
-+ }
-+
-+ /* we have 2-disk failure */
-+ BUG_ON(s->failed != 2);
-+ /* fall through */
-+ case check_state_compute_result:
-+ sh->check_state = check_state_idle;
-+
-+ /* check that a write has not made the stripe insync */
-+ if (test_bit(STRIPE_INSYNC, &sh->state))
-+ break;
-+
-+ /* now write out any block on a failed drive,
-+ * or P or Q if they were recomputed
-+ */
-+ BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
-+ if (s->failed == 2) {
-+ dev = &sh->dev[s->failed_num[1]];
-+ s->locked++;
-+ set_bit(R5_LOCKED, &dev->flags);
-+ set_bit(R5_Wantwrite, &dev->flags);
-+ }
-+ if (s->failed >= 1) {
-+ dev = &sh->dev[s->failed_num[0]];
-+ s->locked++;
-+ set_bit(R5_LOCKED, &dev->flags);
-+ set_bit(R5_Wantwrite, &dev->flags);
-+ }
-+ if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
-+ dev = &sh->dev[pd_idx];
-+ s->locked++;
-+ set_bit(R5_LOCKED, &dev->flags);
-+ set_bit(R5_Wantwrite, &dev->flags);
-+ }
-+ if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
-+ dev = &sh->dev[qd_idx];
-+ s->locked++;
-+ set_bit(R5_LOCKED, &dev->flags);
-+ set_bit(R5_Wantwrite, &dev->flags);
-+ }
-+ clear_bit(STRIPE_DEGRADED, &sh->state);
-+
-+ set_bit(STRIPE_INSYNC, &sh->state);
-+ break;
-+ case check_state_run:
-+ case check_state_run_q:
-+ case check_state_run_pq:
-+ break; /* we will be called again upon completion */
-+ case check_state_check_result:
-+ sh->check_state = check_state_idle;
-+
-+ /* handle a successful check operation, if parity is correct
-+ * we are done. Otherwise update the mismatch count and repair
-+ * parity if !MD_RECOVERY_CHECK
-+ */
-+ if (sh->ops.zero_sum_result == 0) {
-+ /* both parities are correct */
-+ if (!s->failed)
-+ set_bit(STRIPE_INSYNC, &sh->state);
-+ else {
-+ /* in contrast to the raid5 case we can validate
-+ * parity, but still have a failure to write
-+ * back
-+ */
-+ sh->check_state = check_state_compute_result;
-+ /* Returning at this point means that we may go
-+ * off and bring p and/or q uptodate again so
-+ * we make sure to check zero_sum_result again
-+ * to verify if p or q need writeback
-+ */
-+ }
-+ } else {
-+ atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
-+ if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
-+ /* don't try to repair!! */
-+ set_bit(STRIPE_INSYNC, &sh->state);
-+ else {
-+ int *target = &sh->ops.target;
-+
-+ sh->ops.target = -1;
-+ sh->ops.target2 = -1;
-+ sh->check_state = check_state_compute_run;
-+ set_bit(STRIPE_COMPUTE_RUN, &sh->state);
-+ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
-+ if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
-+ set_bit(R5_Wantcompute,
-+ &sh->dev[pd_idx].flags);
-+ *target = pd_idx;
-+ target = &sh->ops.target2;
-+ s->uptodate++;
-+ }
-+ if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
-+ set_bit(R5_Wantcompute,
-+ &sh->dev[qd_idx].flags);
-+ *target = qd_idx;
-+ s->uptodate++;
-+ }
-+ }
-+ }
-+ break;
-+ case check_state_compute_run:
-+ break;
-+ default:
-+ printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
-+ __func__, sh->check_state,
-+ (unsigned long long) sh->sector);
-+ BUG();
-+ }
-+}
-+
-+static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
-+{
-+ int i;
-+
-+ /* We have read all the blocks in this stripe and now we need to
-+ * copy some of them into a target stripe for expand.
-+ */
-+ struct dma_async_tx_descriptor *tx = NULL;
-+ BUG_ON(sh->batch_head);
-+ clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
-+ for (i = 0; i < sh->disks; i++)
-+ if (i != sh->pd_idx && i != sh->qd_idx) {
-+ int dd_idx, j;
-+ struct stripe_head *sh2;
-+ struct async_submit_ctl submit;
-+
-+ sector_t bn = compute_blocknr(sh, i, 1);
-+ sector_t s = raid5_compute_sector(conf, bn, 0,
-+ &dd_idx, NULL);
-+ sh2 = get_active_stripe(conf, s, 0, 1, 1);
-+ if (sh2 == NULL)
-+ /* so far only the early blocks of this stripe
-+ * have been requested. When later blocks
-+ * get requested, we will try again
-+ */
-+ continue;
-+ if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
-+ test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
-+ /* must have already done this block */
-+ release_stripe(sh2);
-+ continue;
-+ }
-+
-+ /* place all the copies on one channel */
-+ init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
-+ tx = async_memcpy(sh2->dev[dd_idx].page,
-+ sh->dev[i].page, 0, 0, STRIPE_SIZE,
-+ &submit);
-+
-+ set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
-+ set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
-+ for (j = 0; j < conf->raid_disks; j++)
-+ if (j != sh2->pd_idx &&
-+ j != sh2->qd_idx &&
-+ !test_bit(R5_Expanded, &sh2->dev[j].flags))
-+ break;
-+ if (j == conf->raid_disks) {
-+ set_bit(STRIPE_EXPAND_READY, &sh2->state);
-+ set_bit(STRIPE_HANDLE, &sh2->state);
-+ }
-+ release_stripe(sh2);
-+
-+ }
-+ /* done submitting copies, wait for them to complete */
-+ async_tx_quiesce(&tx);
-+}
-+
-+/*
-+ * handle_stripe - do things to a stripe.
-+ *
-+ * We lock the stripe by setting STRIPE_ACTIVE and then examine the
-+ * state of various bits to see what needs to be done.
-+ * Possible results:
-+ * return some read requests which now have data
-+ * return some write requests which are safely on storage
-+ * schedule a read on some buffers
-+ * schedule a write of some buffers
-+ * return confirmation of parity correctness
-+ *
-+ */
-+
-+static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
-+{
-+ struct r5conf *conf = sh->raid_conf;
-+ int disks = sh->disks;
-+ struct r5dev *dev;
-+ int i;
-+ int do_recovery = 0;
-+
-+ memset(s, 0, sizeof(*s));
-+
-+ s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;
-+ s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;
-+ s->failed_num[0] = -1;
-+ s->failed_num[1] = -1;
-+
-+ /* Now to look around and see what can be done */
-+ rcu_read_lock();
-+ for (i=disks; i--; ) {
-+ struct md_rdev *rdev;
-+ sector_t first_bad;
-+ int bad_sectors;
-+ int is_bad = 0;
-+
-+ dev = &sh->dev[i];
-+
-+ pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
-+ i, dev->flags,
-+ dev->toread, dev->towrite, dev->written);
-+ /* maybe we can reply to a read
-+ *
-+ * new wantfill requests are only permitted while
-+ * ops_complete_biofill is guaranteed to be inactive
-+ */
-+ if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
-+ !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
-+ set_bit(R5_Wantfill, &dev->flags);
-+
-+ /* now count some things */
-+ if (test_bit(R5_LOCKED, &dev->flags))
-+ s->locked++;
-+ if (test_bit(R5_UPTODATE, &dev->flags))
-+ s->uptodate++;
-+ if (test_bit(R5_Wantcompute, &dev->flags)) {
-+ s->compute++;
-+ BUG_ON(s->compute > 2);
-+ }
-+
-+ if (test_bit(R5_Wantfill, &dev->flags))
-+ s->to_fill++;
-+ else if (dev->toread)
-+ s->to_read++;
-+ if (dev->towrite) {
-+ s->to_write++;
-+ if (!test_bit(R5_OVERWRITE, &dev->flags))
-+ s->non_overwrite++;
-+ }
-+ if (dev->written)
-+ s->written++;
-+ /* Prefer to use the replacement for reads, but only
-+ * if it is recovered enough and has no bad blocks.
-+ */
-+ rdev = rcu_dereference(conf->disks[i].replacement);
-+ if (rdev && !test_bit(Faulty, &rdev->flags) &&
-+ rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
-+ !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
-+ &first_bad, &bad_sectors))
-+ set_bit(R5_ReadRepl, &dev->flags);
-+ else {
-+ if (rdev)
-+ set_bit(R5_NeedReplace, &dev->flags);
-+ rdev = rcu_dereference(conf->disks[i].rdev);
-+ clear_bit(R5_ReadRepl, &dev->flags);
-+ }
-+ if (rdev && test_bit(Faulty, &rdev->flags))
-+ rdev = NULL;
-+ if (rdev) {
-+ is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
-+ &first_bad, &bad_sectors);
-+ if (s->blocked_rdev == NULL
-+ && (test_bit(Blocked, &rdev->flags)
-+ || is_bad < 0)) {
-+ if (is_bad < 0)
-+ set_bit(BlockedBadBlocks,
-+ &rdev->flags);
-+ s->blocked_rdev = rdev;
-+ atomic_inc(&rdev->nr_pending);
-+ }
-+ }
-+ clear_bit(R5_Insync, &dev->flags);
-+ if (!rdev)
-+ /* Not in-sync */;
-+ else if (is_bad) {
-+ /* also not in-sync */
-+ if (!test_bit(WriteErrorSeen, &rdev->flags) &&
-+ test_bit(R5_UPTODATE, &dev->flags)) {
-+ /* treat as in-sync, but with a read error
-+ * which we can now try to correct
-+ */
-+ set_bit(R5_Insync, &dev->flags);
-+ set_bit(R5_ReadError, &dev->flags);
-+ }
-+ } else if (test_bit(In_sync, &rdev->flags))
-+ set_bit(R5_Insync, &dev->flags);
-+ else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
-+ /* in sync if before recovery_offset */
-+ set_bit(R5_Insync, &dev->flags);
-+ else if (test_bit(R5_UPTODATE, &dev->flags) &&
-+ test_bit(R5_Expanded, &dev->flags))
-+ /* If we've reshaped into here, we assume it is Insync.
-+ * We will shortly update recovery_offset to make
-+ * it official.
-+ */
-+ set_bit(R5_Insync, &dev->flags);
-+
-+ if (test_bit(R5_WriteError, &dev->flags)) {
-+ /* This flag does not apply to '.replacement'
-+ * only to .rdev, so make sure to check that*/
-+ struct md_rdev *rdev2 = rcu_dereference(
-+ conf->disks[i].rdev);
-+ if (rdev2 == rdev)
-+ clear_bit(R5_Insync, &dev->flags);
-+ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
-+ s->handle_bad_blocks = 1;
-+ atomic_inc(&rdev2->nr_pending);
-+ } else
-+ clear_bit(R5_WriteError, &dev->flags);
-+ }
-+ if (test_bit(R5_MadeGood, &dev->flags)) {
-+ /* This flag does not apply to '.replacement'
-+ * only to .rdev, so make sure to check that*/
-+ struct md_rdev *rdev2 = rcu_dereference(
-+ conf->disks[i].rdev);
-+ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
-+ s->handle_bad_blocks = 1;
-+ atomic_inc(&rdev2->nr_pending);
-+ } else
-+ clear_bit(R5_MadeGood, &dev->flags);
-+ }
-+ if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
-+ struct md_rdev *rdev2 = rcu_dereference(
-+ conf->disks[i].replacement);
-+ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
-+ s->handle_bad_blocks = 1;
-+ atomic_inc(&rdev2->nr_pending);
-+ } else
-+ clear_bit(R5_MadeGoodRepl, &dev->flags);
-+ }
-+ if (!test_bit(R5_Insync, &dev->flags)) {
-+ /* The ReadError flag will just be confusing now */
-+ clear_bit(R5_ReadError, &dev->flags);
-+ clear_bit(R5_ReWrite, &dev->flags);
-+ }
-+ if (test_bit(R5_ReadError, &dev->flags))
-+ clear_bit(R5_Insync, &dev->flags);
-+ if (!test_bit(R5_Insync, &dev->flags)) {
-+ if (s->failed < 2)
-+ s->failed_num[s->failed] = i;
-+ s->failed++;
-+ if (rdev && !test_bit(Faulty, &rdev->flags))
-+ do_recovery = 1;
-+ }
-+ }
-+ if (test_bit(STRIPE_SYNCING, &sh->state)) {
-+ /* If there is a failed device being replaced,
-+ * we must be recovering.
-+ * else if we are after recovery_cp, we must be syncing
-+ * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
-+ * else we can only be replacing
-+ * sync and recovery both need to read all devices, and so
-+ * use the same flag.
-+ */
-+ if (do_recovery ||
-+ sh->sector >= conf->mddev->recovery_cp ||
-+ test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
-+ s->syncing = 1;
-+ else
-+ s->replacing = 1;
-+ }
-+ rcu_read_unlock();
-+}
-+
-+static int clear_batch_ready(struct stripe_head *sh)
-+{
-+ /* Return '1' if this is a member of batch, or
-+ * '0' if it is a lone stripe or a head which can now be
-+ * handled.
-+ */
-+ struct stripe_head *tmp;
-+ if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
-+ return (sh->batch_head && sh->batch_head != sh);
-+ spin_lock(&sh->stripe_lock);
-+ if (!sh->batch_head) {
-+ spin_unlock(&sh->stripe_lock);
-+ return 0;
-+ }
-+
-+ /*
-+ * this stripe could be added to a batch list before we check
-+ * BATCH_READY, skips it
-+ */
-+ if (sh->batch_head != sh) {
-+ spin_unlock(&sh->stripe_lock);
-+ return 1;
-+ }
-+ spin_lock(&sh->batch_lock);
-+ list_for_each_entry(tmp, &sh->batch_list, batch_list)
-+ clear_bit(STRIPE_BATCH_READY, &tmp->state);
-+ spin_unlock(&sh->batch_lock);
-+ spin_unlock(&sh->stripe_lock);
-+
-+ /*
-+ * BATCH_READY is cleared, no new stripes can be added.
-+ * batch_list can be accessed without lock
-+ */
-+ return 0;
-+}
-+
-+static void break_stripe_batch_list(struct stripe_head *head_sh,
-+ unsigned long handle_flags)
-+{
-+ struct stripe_head *sh, *next;
-+ int i;
-+ int do_wakeup = 0;
-+
-+ list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {
-+
-+ list_del_init(&sh->batch_list);
-+
-+ WARN_ON_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |
-+ (1 << STRIPE_SYNCING) |
-+ (1 << STRIPE_REPLACED) |
-+ (1 << STRIPE_PREREAD_ACTIVE) |
-+ (1 << STRIPE_DELAYED) |
-+ (1 << STRIPE_BIT_DELAY) |
-+ (1 << STRIPE_FULL_WRITE) |
-+ (1 << STRIPE_BIOFILL_RUN) |
-+ (1 << STRIPE_COMPUTE_RUN) |
-+ (1 << STRIPE_OPS_REQ_PENDING) |
-+ (1 << STRIPE_DISCARD) |
-+ (1 << STRIPE_BATCH_READY) |
-+ (1 << STRIPE_BATCH_ERR) |
-+ (1 << STRIPE_BITMAP_PENDING)));
-+ WARN_ON_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) |
-+ (1 << STRIPE_REPLACED)));
-+
-+ set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |
-+ (1 << STRIPE_DEGRADED)),
-+ head_sh->state & (1 << STRIPE_INSYNC));
-+
-+ sh->check_state = head_sh->check_state;
-+ sh->reconstruct_state = head_sh->reconstruct_state;
-+ for (i = 0; i < sh->disks; i++) {
-+ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
-+ do_wakeup = 1;
-+ sh->dev[i].flags = head_sh->dev[i].flags &
-+ (~((1 << R5_WriteError) | (1 << R5_Overlap)));
-+ }
-+ spin_lock_irq(&sh->stripe_lock);
-+ sh->batch_head = NULL;
-+ spin_unlock_irq(&sh->stripe_lock);
-+ if (handle_flags == 0 ||
-+ sh->state & handle_flags)
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ release_stripe(sh);
-+ }
-+ spin_lock_irq(&head_sh->stripe_lock);
-+ head_sh->batch_head = NULL;
-+ spin_unlock_irq(&head_sh->stripe_lock);
-+ for (i = 0; i < head_sh->disks; i++)
-+ if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags))
-+ do_wakeup = 1;
-+ if (head_sh->state & handle_flags)
-+ set_bit(STRIPE_HANDLE, &head_sh->state);
-+
-+ if (do_wakeup)
-+ wake_up(&head_sh->raid_conf->wait_for_overlap);
-+}
-+
-+static void handle_stripe(struct stripe_head *sh)
-+{
-+ struct stripe_head_state s;
-+ struct r5conf *conf = sh->raid_conf;
-+ int i;
-+ int prexor;
-+ int disks = sh->disks;
-+ struct r5dev *pdev, *qdev;
-+
-+ clear_bit(STRIPE_HANDLE, &sh->state);
-+ if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
-+ /* already being handled, ensure it gets handled
-+ * again when current action finishes */
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ return;
-+ }
-+
-+ if (clear_batch_ready(sh) ) {
-+ clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
-+ return;
-+ }
-+
-+ if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
-+ break_stripe_batch_list(sh, 0);
-+
-+ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {
-+ spin_lock(&sh->stripe_lock);
-+ /* Cannot process 'sync' concurrently with 'discard' */
-+ if (!test_bit(STRIPE_DISCARD, &sh->state) &&
-+ test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
-+ set_bit(STRIPE_SYNCING, &sh->state);
-+ clear_bit(STRIPE_INSYNC, &sh->state);
-+ clear_bit(STRIPE_REPLACED, &sh->state);
-+ }
-+ spin_unlock(&sh->stripe_lock);
-+ }
-+ clear_bit(STRIPE_DELAYED, &sh->state);
-+
-+ pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
-+ "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
-+ (unsigned long long)sh->sector, sh->state,
-+ atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
-+ sh->check_state, sh->reconstruct_state);
-+
-+ analyse_stripe(sh, &s);
-+
-+ if (s.handle_bad_blocks) {
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ goto finish;
-+ }
-+
-+ if (unlikely(s.blocked_rdev)) {
-+ if (s.syncing || s.expanding || s.expanded ||
-+ s.replacing || s.to_write || s.written) {
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ goto finish;
-+ }
-+ /* There is nothing for the blocked_rdev to block */
-+ rdev_dec_pending(s.blocked_rdev, conf->mddev);
-+ s.blocked_rdev = NULL;
-+ }
-+
-+ if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
-+ set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
-+ set_bit(STRIPE_BIOFILL_RUN, &sh->state);
-+ }
-+
-+ pr_debug("locked=%d uptodate=%d to_read=%d"
-+ " to_write=%d failed=%d failed_num=%d,%d\n",
-+ s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
-+ s.failed_num[0], s.failed_num[1]);
-+ /* check if the array has lost more than max_degraded devices and,
-+ * if so, some requests might need to be failed.
-+ */
-+ if (s.failed > conf->max_degraded) {
-+ sh->check_state = 0;
-+ sh->reconstruct_state = 0;
-+ break_stripe_batch_list(sh, 0);
-+ if (s.to_read+s.to_write+s.written)
-+ handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
-+ if (s.syncing + s.replacing)
-+ handle_failed_sync(conf, sh, &s);
-+ }
-+
-+ /* Now we check to see if any write operations have recently
-+ * completed
-+ */
-+ prexor = 0;
-+ if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
-+ prexor = 1;
-+ if (sh->reconstruct_state == reconstruct_state_drain_result ||
-+ sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
-+ sh->reconstruct_state = reconstruct_state_idle;
-+
-+ /* All the 'written' buffers and the parity block are ready to
-+ * be written back to disk
-+ */
-+ BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
-+ !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
-+ BUG_ON(sh->qd_idx >= 0 &&
-+ !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
-+ !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
-+ for (i = disks; i--; ) {
-+ struct r5dev *dev = &sh->dev[i];
-+ if (test_bit(R5_LOCKED, &dev->flags) &&
-+ (i == sh->pd_idx || i == sh->qd_idx ||
-+ dev->written)) {
-+ pr_debug("Writing block %d\n", i);
-+ set_bit(R5_Wantwrite, &dev->flags);
-+ if (prexor)
-+ continue;
-+ if (s.failed > 1)
-+ continue;
-+ if (!test_bit(R5_Insync, &dev->flags) ||
-+ ((i == sh->pd_idx || i == sh->qd_idx) &&
-+ s.failed == 0))
-+ set_bit(STRIPE_INSYNC, &sh->state);
-+ }
-+ }
-+ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-+ s.dec_preread_active = 1;
-+ }
-+
-+ /*
-+ * might be able to return some write requests if the parity blocks
-+ * are safe, or on a failed drive
-+ */
-+ pdev = &sh->dev[sh->pd_idx];
-+ s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
-+ || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
-+ qdev = &sh->dev[sh->qd_idx];
-+ s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
-+ || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
-+ || conf->level < 6;
-+
-+ if (s.written &&
-+ (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
-+ && !test_bit(R5_LOCKED, &pdev->flags)
-+ && (test_bit(R5_UPTODATE, &pdev->flags) ||
-+ test_bit(R5_Discard, &pdev->flags))))) &&
-+ (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
-+ && !test_bit(R5_LOCKED, &qdev->flags)
-+ && (test_bit(R5_UPTODATE, &qdev->flags) ||
-+ test_bit(R5_Discard, &qdev->flags))))))
-+ handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
-+
-+ /* Now we might consider reading some blocks, either to check/generate
-+ * parity, or to satisfy requests
-+ * or to load a block that is being partially written.
-+ */
-+ if (s.to_read || s.non_overwrite
-+ || (conf->level == 6 && s.to_write && s.failed)
-+ || (s.syncing && (s.uptodate + s.compute < disks))
-+ || s.replacing
-+ || s.expanding)
-+ handle_stripe_fill(sh, &s, disks);
-+
-+ /* Now to consider new write requests and what else, if anything
-+ * should be read. We do not handle new writes when:
-+ * 1/ A 'write' operation (copy+xor) is already in flight.
-+ * 2/ A 'check' operation is in flight, as it may clobber the parity
-+ * block.
-+ */
-+ if (s.to_write && !sh->reconstruct_state && !sh->check_state)
-+ handle_stripe_dirtying(conf, sh, &s, disks);
-+
-+ /* maybe we need to check and possibly fix the parity for this stripe
-+ * Any reads will already have been scheduled, so we just see if enough
-+ * data is available. The parity check is held off while parity
-+ * dependent operations are in flight.
-+ */
-+ if (sh->check_state ||
-+ (s.syncing && s.locked == 0 &&
-+ !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
-+ !test_bit(STRIPE_INSYNC, &sh->state))) {
-+ if (conf->level == 6)
-+ handle_parity_checks6(conf, sh, &s, disks);
-+ else
-+ handle_parity_checks5(conf, sh, &s, disks);
-+ }
-+
-+ if ((s.replacing || s.syncing) && s.locked == 0
-+ && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
-+ && !test_bit(STRIPE_REPLACED, &sh->state)) {
-+ /* Write out to replacement devices where possible */
-+ for (i = 0; i < conf->raid_disks; i++)
-+ if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
-+ WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
-+ set_bit(R5_WantReplace, &sh->dev[i].flags);
-+ set_bit(R5_LOCKED, &sh->dev[i].flags);
-+ s.locked++;
-+ }
-+ if (s.replacing)
-+ set_bit(STRIPE_INSYNC, &sh->state);
-+ set_bit(STRIPE_REPLACED, &sh->state);
-+ }
-+ if ((s.syncing || s.replacing) && s.locked == 0 &&
-+ !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
-+ test_bit(STRIPE_INSYNC, &sh->state)) {
-+ md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
-+ clear_bit(STRIPE_SYNCING, &sh->state);
-+ if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
-+ wake_up(&conf->wait_for_overlap);
-+ }
-+
-+ /* If the failed drives are just a ReadError, then we might need
-+ * to progress the repair/check process
-+ */
-+ if (s.failed <= conf->max_degraded && !conf->mddev->ro)
-+ for (i = 0; i < s.failed; i++) {
-+ struct r5dev *dev = &sh->dev[s.failed_num[i]];
-+ if (test_bit(R5_ReadError, &dev->flags)
-+ && !test_bit(R5_LOCKED, &dev->flags)
-+ && test_bit(R5_UPTODATE, &dev->flags)
-+ ) {
-+ if (!test_bit(R5_ReWrite, &dev->flags)) {
-+ set_bit(R5_Wantwrite, &dev->flags);
-+ set_bit(R5_ReWrite, &dev->flags);
-+ set_bit(R5_LOCKED, &dev->flags);
-+ s.locked++;
-+ } else {
-+ /* let's read it back */
-+ set_bit(R5_Wantread, &dev->flags);
-+ set_bit(R5_LOCKED, &dev->flags);
-+ s.locked++;
-+ }
-+ }
-+ }
-+
-+ /* Finish reconstruct operations initiated by the expansion process */
-+ if (sh->reconstruct_state == reconstruct_state_result) {
-+ struct stripe_head *sh_src
-+ = get_active_stripe(conf, sh->sector, 1, 1, 1);
-+ if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
-+ /* sh cannot be written until sh_src has been read.
-+ * so arrange for sh to be delayed a little
-+ */
-+ set_bit(STRIPE_DELAYED, &sh->state);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
-+ &sh_src->state))
-+ atomic_inc(&conf->preread_active_stripes);
-+ release_stripe(sh_src);
-+ goto finish;
-+ }
-+ if (sh_src)
-+ release_stripe(sh_src);
-+
-+ sh->reconstruct_state = reconstruct_state_idle;
-+ clear_bit(STRIPE_EXPANDING, &sh->state);
-+ for (i = conf->raid_disks; i--; ) {
-+ set_bit(R5_Wantwrite, &sh->dev[i].flags);
-+ set_bit(R5_LOCKED, &sh->dev[i].flags);
-+ s.locked++;
-+ }
-+ }
-+
-+ if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
-+ !sh->reconstruct_state) {
-+ /* Need to write out all blocks after computing parity */
-+ sh->disks = conf->raid_disks;
-+ stripe_set_idx(sh->sector, conf, 0, sh);
-+ schedule_reconstruction(sh, &s, 1, 1);
-+ } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
-+ clear_bit(STRIPE_EXPAND_READY, &sh->state);
-+ atomic_dec(&conf->reshape_stripes);
-+ wake_up(&conf->wait_for_overlap);
-+ md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
-+ }
-+
-+ if (s.expanding && s.locked == 0 &&
-+ !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
-+ handle_stripe_expansion(conf, sh);
-+
-+finish:
-+ /* wait for this device to become unblocked */
-+ if (unlikely(s.blocked_rdev)) {
-+ if (conf->mddev->external)
-+ md_wait_for_blocked_rdev(s.blocked_rdev,
-+ conf->mddev);
-+ else
-+ /* Internal metadata will immediately
-+ * be written by raid5d, so we don't
-+ * need to wait here.
-+ */
-+ rdev_dec_pending(s.blocked_rdev,
-+ conf->mddev);
-+ }
-+
-+ if (s.handle_bad_blocks)
-+ for (i = disks; i--; ) {
-+ struct md_rdev *rdev;
-+ struct r5dev *dev = &sh->dev[i];
-+ if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
-+ /* We own a safe reference to the rdev */
-+ rdev = conf->disks[i].rdev;
-+ if (!rdev_set_badblocks(rdev, sh->sector,
-+ STRIPE_SECTORS, 0))
-+ md_error(conf->mddev, rdev);
-+ rdev_dec_pending(rdev, conf->mddev);
-+ }
-+ if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
-+ rdev = conf->disks[i].rdev;
-+ rdev_clear_badblocks(rdev, sh->sector,
-+ STRIPE_SECTORS, 0);
-+ rdev_dec_pending(rdev, conf->mddev);
-+ }
-+ if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
-+ rdev = conf->disks[i].replacement;
-+ if (!rdev)
-+ /* rdev have been moved down */
-+ rdev = conf->disks[i].rdev;
-+ rdev_clear_badblocks(rdev, sh->sector,
-+ STRIPE_SECTORS, 0);
-+ rdev_dec_pending(rdev, conf->mddev);
-+ }
-+ }
-+
-+ if (s.ops_request)
-+ raid_run_ops(sh, s.ops_request);
-+
-+ ops_run_io(sh, &s);
-+
-+ if (s.dec_preread_active) {
-+ /* We delay this until after ops_run_io so that if make_request
-+ * is waiting on a flush, it won't continue until the writes
-+ * have actually been submitted.
-+ */
-+ atomic_dec(&conf->preread_active_stripes);
-+ if (atomic_read(&conf->preread_active_stripes) <
-+ IO_THRESHOLD)
-+ md_wakeup_thread(conf->mddev->thread);
-+ }
-+
-+ return_io(s.return_bi);
-+
-+ clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
-+}
-+
-+static void raid5_activate_delayed(struct r5conf *conf)
-+{
-+ if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
-+ while (!list_empty(&conf->delayed_list)) {
-+ struct list_head *l = conf->delayed_list.next;
-+ struct stripe_head *sh;
-+ sh = list_entry(l, struct stripe_head, lru);
-+ list_del_init(l);
-+ clear_bit(STRIPE_DELAYED, &sh->state);
-+ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-+ atomic_inc(&conf->preread_active_stripes);
-+ list_add_tail(&sh->lru, &conf->hold_list);
-+ raid5_wakeup_stripe_thread(sh);
-+ }
-+ }
-+}
-+
-+static void activate_bit_delay(struct r5conf *conf,
-+ struct list_head *temp_inactive_list)
-+{
-+ /* device_lock is held */
-+ struct list_head head;
-+ list_add(&head, &conf->bitmap_list);
-+ list_del_init(&conf->bitmap_list);
-+ while (!list_empty(&head)) {
-+ struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
-+ int hash;
-+ list_del_init(&sh->lru);
-+ atomic_inc(&sh->count);
-+ hash = sh->hash_lock_index;
-+ __release_stripe(conf, sh, &temp_inactive_list[hash]);
-+ }
-+}
-+
-+static int raid5_congested(struct mddev *mddev, int bits)
-+{
-+ struct r5conf *conf = mddev->private;
-+
-+ /* No difference between reads and writes. Just check
-+ * how busy the stripe_cache is
-+ */
-+
-+ if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
-+ return 1;
-+ if (conf->quiesce)
-+ return 1;
-+ if (atomic_read(&conf->empty_inactive_list_nr))
-+ return 1;
-+
-+ return 0;
-+}
-+
-+/* We want read requests to align with chunks where possible,
-+ * but write requests don't need to.
-+ */
-+static int raid5_mergeable_bvec(struct mddev *mddev,
-+ struct bvec_merge_data *bvm,
-+ struct bio_vec *biovec)
-+{
-+ sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
-+ int max;
-+ unsigned int chunk_sectors = mddev->chunk_sectors;
-+ unsigned int bio_sectors = bvm->bi_size >> 9;
-+
-+ /*
-+ * always allow writes to be mergeable, read as well if array
-+ * is degraded as we'll go through stripe cache anyway.
-+ */
-+ if ((bvm->bi_rw & 1) == WRITE || mddev->degraded)
-+ return biovec->bv_len;
-+
-+ if (mddev->new_chunk_sectors < mddev->chunk_sectors)
-+ chunk_sectors = mddev->new_chunk_sectors;
-+ max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
-+ if (max < 0) max = 0;
-+ if (max <= biovec->bv_len && bio_sectors == 0)
-+ return biovec->bv_len;
-+ else
-+ return max;
-+}
-+
-+static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
-+{
-+ sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev);
-+ unsigned int chunk_sectors = mddev->chunk_sectors;
-+ unsigned int bio_sectors = bio_sectors(bio);
-+
-+ if (mddev->new_chunk_sectors < mddev->chunk_sectors)
-+ chunk_sectors = mddev->new_chunk_sectors;
-+ return chunk_sectors >=
-+ ((sector & (chunk_sectors - 1)) + bio_sectors);
-+}
-+
-+/*
-+ * add bio to the retry LIFO ( in O(1) ... we are in interrupt )
-+ * later sampled by raid5d.
-+ */
-+static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
-+{
-+ unsigned long flags;
-+
-+ spin_lock_irqsave(&conf->device_lock, flags);
-+
-+ bi->bi_next = conf->retry_read_aligned_list;
-+ conf->retry_read_aligned_list = bi;
-+
-+ spin_unlock_irqrestore(&conf->device_lock, flags);
-+ md_wakeup_thread(conf->mddev->thread);
-+}
-+
-+static struct bio *remove_bio_from_retry(struct r5conf *conf)
-+{
-+ struct bio *bi;
-+
-+ bi = conf->retry_read_aligned;
-+ if (bi) {
-+ conf->retry_read_aligned = NULL;
-+ return bi;
-+ }
-+ bi = conf->retry_read_aligned_list;
-+ if(bi) {
-+ conf->retry_read_aligned_list = bi->bi_next;
-+ bi->bi_next = NULL;
-+ /*
-+ * this sets the active strip count to 1 and the processed
-+ * strip count to zero (upper 8 bits)
-+ */
-+ raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
-+ }
-+
-+ return bi;
-+}
-+
-+/*
-+ * The "raid5_align_endio" should check if the read succeeded and if it
-+ * did, call bio_endio on the original bio (having bio_put the new bio
-+ * first).
-+ * If the read failed..
-+ */
-+static void raid5_align_endio(struct bio *bi, int error)
-+{
-+ struct bio* raid_bi = bi->bi_private;
-+ struct mddev *mddev;
-+ struct r5conf *conf;
-+ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
-+ struct md_rdev *rdev;
-+
-+ bio_put(bi);
-+
-+ rdev = (void*)raid_bi->bi_next;
-+ raid_bi->bi_next = NULL;
-+ mddev = rdev->mddev;
-+ conf = mddev->private;
-+
-+ rdev_dec_pending(rdev, conf->mddev);
-+
-+ if (!error && uptodate) {
-+ trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev),
-+ raid_bi, 0);
-+ bio_endio(raid_bi, 0);
-+ if (atomic_dec_and_test(&conf->active_aligned_reads))
-+ wake_up(&conf->wait_for_stripe);
-+ return;
-+ }
-+
-+ pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
-+
-+ add_bio_to_retry(raid_bi, conf);
-+}
-+
-+static int bio_fits_rdev(struct bio *bi)
-+{
-+ struct request_queue *q = bdev_get_queue(bi->bi_bdev);
-+
-+ if (bio_sectors(bi) > queue_max_sectors(q))
-+ return 0;
-+ blk_recount_segments(q, bi);
-+ if (bi->bi_phys_segments > queue_max_segments(q))
-+ return 0;
-+
-+ if (q->merge_bvec_fn)
-+ /* it's too hard to apply the merge_bvec_fn at this stage,
-+ * just just give up
-+ */
-+ return 0;
-+
-+ return 1;
-+}
-+
-+static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
-+{
-+ struct r5conf *conf = mddev->private;
-+ int dd_idx;
-+ struct bio* align_bi;
-+ struct md_rdev *rdev;
-+ sector_t end_sector;
-+
-+ if (!in_chunk_boundary(mddev, raid_bio)) {
-+ pr_debug("chunk_aligned_read : non aligned\n");
-+ return 0;
-+ }
-+ /*
-+ * use bio_clone_mddev to make a copy of the bio
-+ */
-+ align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
-+ if (!align_bi)
-+ return 0;
-+ /*
-+ * set bi_end_io to a new function, and set bi_private to the
-+ * original bio.
-+ */
-+ align_bi->bi_end_io = raid5_align_endio;
-+ align_bi->bi_private = raid_bio;
-+ /*
-+ * compute position
-+ */
-+ align_bi->bi_iter.bi_sector =
-+ raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
-+ 0, &dd_idx, NULL);
-+
-+ end_sector = bio_end_sector(align_bi);
-+ rcu_read_lock();
-+ rdev = rcu_dereference(conf->disks[dd_idx].replacement);
-+ if (!rdev || test_bit(Faulty, &rdev->flags) ||
-+ rdev->recovery_offset < end_sector) {
-+ rdev = rcu_dereference(conf->disks[dd_idx].rdev);
-+ if (rdev &&
-+ (test_bit(Faulty, &rdev->flags) ||
-+ !(test_bit(In_sync, &rdev->flags) ||
-+ rdev->recovery_offset >= end_sector)))
-+ rdev = NULL;
-+ }
-+ if (rdev) {
-+ sector_t first_bad;
-+ int bad_sectors;
-+
-+ atomic_inc(&rdev->nr_pending);
-+ rcu_read_unlock();
-+ raid_bio->bi_next = (void*)rdev;
-+ align_bi->bi_bdev = rdev->bdev;
-+ __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags);
-+
-+ if (!bio_fits_rdev(align_bi) ||
-+ is_badblock(rdev, align_bi->bi_iter.bi_sector,
-+ bio_sectors(align_bi),
-+ &first_bad, &bad_sectors)) {
-+ /* too big in some way, or has a known bad block */
-+ bio_put(align_bi);
-+ rdev_dec_pending(rdev, mddev);
-+ return 0;
-+ }
-+
-+ /* No reshape active, so we can trust rdev->data_offset */
-+ align_bi->bi_iter.bi_sector += rdev->data_offset;
-+
-+ spin_lock_irq(&conf->device_lock);
-+ wait_event_lock_irq(conf->wait_for_stripe,
-+ conf->quiesce == 0,
-+ conf->device_lock);
-+ atomic_inc(&conf->active_aligned_reads);
-+ spin_unlock_irq(&conf->device_lock);
-+
-+ if (mddev->gendisk)
-+ trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
-+ align_bi, disk_devt(mddev->gendisk),
-+ raid_bio->bi_iter.bi_sector);
-+ generic_make_request(align_bi);
-+ return 1;
-+ } else {
-+ rcu_read_unlock();
-+ bio_put(align_bi);
-+ return 0;
-+ }
-+}
-+
-+/* __get_priority_stripe - get the next stripe to process
-+ *
-+ * Full stripe writes are allowed to pass preread active stripes up until
-+ * the bypass_threshold is exceeded. In general the bypass_count
-+ * increments when the handle_list is handled before the hold_list; however, it
-+ * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
-+ * stripe with in flight i/o. The bypass_count will be reset when the
-+ * head of the hold_list has changed, i.e. the head was promoted to the
-+ * handle_list.
-+ */
-+static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
-+{
-+ struct stripe_head *sh = NULL, *tmp;
-+ struct list_head *handle_list = NULL;
-+ struct r5worker_group *wg = NULL;
-+
-+ if (conf->worker_cnt_per_group == 0) {
-+ handle_list = &conf->handle_list;
-+ } else if (group != ANY_GROUP) {
-+ handle_list = &conf->worker_groups[group].handle_list;
-+ wg = &conf->worker_groups[group];
-+ } else {
-+ int i;
-+ for (i = 0; i < conf->group_cnt; i++) {
-+ handle_list = &conf->worker_groups[i].handle_list;
-+ wg = &conf->worker_groups[i];
-+ if (!list_empty(handle_list))
-+ break;
-+ }
-+ }
-+
-+ pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
-+ __func__,
-+ list_empty(handle_list) ? "empty" : "busy",
-+ list_empty(&conf->hold_list) ? "empty" : "busy",
-+ atomic_read(&conf->pending_full_writes), conf->bypass_count);
-+
-+ if (!list_empty(handle_list)) {
-+ sh = list_entry(handle_list->next, typeof(*sh), lru);
-+
-+ if (list_empty(&conf->hold_list))
-+ conf->bypass_count = 0;
-+ else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
-+ if (conf->hold_list.next == conf->last_hold)
-+ conf->bypass_count++;
-+ else {
-+ conf->last_hold = conf->hold_list.next;
-+ conf->bypass_count -= conf->bypass_threshold;
-+ if (conf->bypass_count < 0)
-+ conf->bypass_count = 0;
-+ }
-+ }
-+ } else if (!list_empty(&conf->hold_list) &&
-+ ((conf->bypass_threshold &&
-+ conf->bypass_count > conf->bypass_threshold) ||
-+ atomic_read(&conf->pending_full_writes) == 0)) {
-+
-+ list_for_each_entry(tmp, &conf->hold_list, lru) {
-+ if (conf->worker_cnt_per_group == 0 ||
-+ group == ANY_GROUP ||
-+ !cpu_online(tmp->cpu) ||
-+ cpu_to_group(tmp->cpu) == group) {
-+ sh = tmp;
-+ break;
-+ }
-+ }
-+
-+ if (sh) {
-+ conf->bypass_count -= conf->bypass_threshold;
-+ if (conf->bypass_count < 0)
-+ conf->bypass_count = 0;
-+ }
-+ wg = NULL;
-+ }
-+
-+ if (!sh)
-+ return NULL;
-+
-+ if (wg) {
-+ wg->stripes_cnt--;
-+ sh->group = NULL;
-+ }
-+ list_del_init(&sh->lru);
-+ BUG_ON(atomic_inc_return(&sh->count) != 1);
-+ return sh;
-+}
-+
-+struct raid5_plug_cb {
-+ struct blk_plug_cb cb;
-+ struct list_head list;
-+ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
-+};
-+
-+static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
-+{
-+ struct raid5_plug_cb *cb = container_of(
-+ blk_cb, struct raid5_plug_cb, cb);
-+ struct stripe_head *sh;
-+ struct mddev *mddev = cb->cb.data;
-+ struct r5conf *conf = mddev->private;
-+ int cnt = 0;
-+ int hash;
-+
-+ if (cb->list.next && !list_empty(&cb->list)) {
-+ spin_lock_irq(&conf->device_lock);
-+ while (!list_empty(&cb->list)) {
-+ sh = list_first_entry(&cb->list, struct stripe_head, lru);
-+ list_del_init(&sh->lru);
-+ /*
-+ * avoid race release_stripe_plug() sees
-+ * STRIPE_ON_UNPLUG_LIST clear but the stripe
-+ * is still in our list
-+ */
-+ smp_mb__before_atomic();
-+ clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
-+ /*
-+ * STRIPE_ON_RELEASE_LIST could be set here. In that
-+ * case, the count is always > 1 here
-+ */
-+ hash = sh->hash_lock_index;
-+ __release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
-+ cnt++;
-+ }
-+ spin_unlock_irq(&conf->device_lock);
-+ }
-+ release_inactive_stripe_list(conf, cb->temp_inactive_list,
-+ NR_STRIPE_HASH_LOCKS);
-+ if (mddev->queue)
-+ trace_block_unplug(mddev->queue, cnt, !from_schedule);
-+ kfree(cb);
-+}
-+
-+static void release_stripe_plug(struct mddev *mddev,
-+ struct stripe_head *sh)
-+{
-+ struct blk_plug_cb *blk_cb = blk_check_plugged(
-+ raid5_unplug, mddev,
-+ sizeof(struct raid5_plug_cb));
-+ struct raid5_plug_cb *cb;
-+
-+ if (!blk_cb) {
-+ release_stripe(sh);
-+ return;
-+ }
-+
-+ cb = container_of(blk_cb, struct raid5_plug_cb, cb);
-+
-+ if (cb->list.next == NULL) {
-+ int i;
-+ INIT_LIST_HEAD(&cb->list);
-+ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
-+ INIT_LIST_HEAD(cb->temp_inactive_list + i);
-+ }
-+
-+ if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
-+ list_add_tail(&sh->lru, &cb->list);
-+ else
-+ release_stripe(sh);
-+}
-+
-+static void make_discard_request(struct mddev *mddev, struct bio *bi)
-+{
-+ struct r5conf *conf = mddev->private;
-+ sector_t logical_sector, last_sector;
-+ struct stripe_head *sh;
-+ int remaining;
-+ int stripe_sectors;
-+
-+ if (mddev->reshape_position != MaxSector)
-+ /* Skip discard while reshape is happening */
-+ return;
-+
-+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
-+ last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
-+
-+ bi->bi_next = NULL;
-+ bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
-+
-+ stripe_sectors = conf->chunk_sectors *
-+ (conf->raid_disks - conf->max_degraded);
-+ logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
-+ stripe_sectors);
-+ sector_div(last_sector, stripe_sectors);
-+
-+ logical_sector *= conf->chunk_sectors;
-+ last_sector *= conf->chunk_sectors;
-+
-+ for (; logical_sector < last_sector;
-+ logical_sector += STRIPE_SECTORS) {
-+ DEFINE_WAIT(w);
-+ int d;
-+ again:
-+ sh = get_active_stripe(conf, logical_sector, 0, 0, 0);
-+ prepare_to_wait(&conf->wait_for_overlap, &w,
-+ TASK_UNINTERRUPTIBLE);
-+ set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
-+ if (test_bit(STRIPE_SYNCING, &sh->state)) {
-+ release_stripe(sh);
-+ schedule();
-+ goto again;
-+ }
-+ clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
-+ spin_lock_irq(&sh->stripe_lock);
-+ for (d = 0; d < conf->raid_disks; d++) {
-+ if (d == sh->pd_idx || d == sh->qd_idx)
-+ continue;
-+ if (sh->dev[d].towrite || sh->dev[d].toread) {
-+ set_bit(R5_Overlap, &sh->dev[d].flags);
-+ spin_unlock_irq(&sh->stripe_lock);
-+ release_stripe(sh);
-+ schedule();
-+ goto again;
-+ }
-+ }
-+ set_bit(STRIPE_DISCARD, &sh->state);
-+ finish_wait(&conf->wait_for_overlap, &w);
-+ sh->overwrite_disks = 0;
-+ for (d = 0; d < conf->raid_disks; d++) {
-+ if (d == sh->pd_idx || d == sh->qd_idx)
-+ continue;
-+ sh->dev[d].towrite = bi;
-+ set_bit(R5_OVERWRITE, &sh->dev[d].flags);
-+ raid5_inc_bi_active_stripes(bi);
-+ sh->overwrite_disks++;
-+ }
-+ spin_unlock_irq(&sh->stripe_lock);
-+ if (conf->mddev->bitmap) {
-+ for (d = 0;
-+ d < conf->raid_disks - conf->max_degraded;
-+ d++)
-+ bitmap_startwrite(mddev->bitmap,
-+ sh->sector,
-+ STRIPE_SECTORS,
-+ 0);
-+ sh->bm_seq = conf->seq_flush + 1;
-+ set_bit(STRIPE_BIT_DELAY, &sh->state);
-+ }
-+
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ clear_bit(STRIPE_DELAYED, &sh->state);
-+ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-+ atomic_inc(&conf->preread_active_stripes);
-+ release_stripe_plug(mddev, sh);
-+ }
-+
-+ remaining = raid5_dec_bi_active_stripes(bi);
-+ if (remaining == 0) {
-+ md_write_end(mddev);
-+ bio_endio(bi, 0);
-+ }
-+}
-+
-+static void make_request(struct mddev *mddev, struct bio * bi)
-+{
-+ struct r5conf *conf = mddev->private;
-+ int dd_idx;
-+ sector_t new_sector;
-+ sector_t logical_sector, last_sector;
-+ struct stripe_head *sh;
-+ const int rw = bio_data_dir(bi);
-+ int remaining;
-+ DEFINE_WAIT(w);
-+ bool do_prepare;
-+
-+ if (unlikely(bi->bi_rw & REQ_FLUSH)) {
-+ md_flush_request(mddev, bi);
-+ return;
-+ }
-+
-+ md_write_start(mddev, bi);
-+
-+ /*
-+ * If array is degraded, better not do chunk aligned read because
-+ * later we might have to read it again in order to reconstruct
-+ * data on failed drives.
-+ */
-+ if (rw == READ && mddev->degraded == 0 &&
-+ mddev->reshape_position == MaxSector &&
-+ chunk_aligned_read(mddev,bi))
-+ return;
-+
-+ if (unlikely(bi->bi_rw & REQ_DISCARD)) {
-+ make_discard_request(mddev, bi);
-+ return;
-+ }
-+
-+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
-+ last_sector = bio_end_sector(bi);
-+ bi->bi_next = NULL;
-+ bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
-+
-+ prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
-+ for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
-+ int previous;
-+ int seq;
-+
-+ do_prepare = false;
-+ retry:
-+ seq = read_seqcount_begin(&conf->gen_lock);
-+ previous = 0;
-+ if (do_prepare)
-+ prepare_to_wait(&conf->wait_for_overlap, &w,
-+ TASK_UNINTERRUPTIBLE);
-+ if (unlikely(conf->reshape_progress != MaxSector)) {
-+ /* spinlock is needed as reshape_progress may be
-+ * 64bit on a 32bit platform, and so it might be
-+ * possible to see a half-updated value
-+ * Of course reshape_progress could change after
-+ * the lock is dropped, so once we get a reference
-+ * to the stripe that we think it is, we will have
-+ * to check again.
-+ */
-+ spin_lock_irq(&conf->device_lock);
-+ if (mddev->reshape_backwards
-+ ? logical_sector < conf->reshape_progress
-+ : logical_sector >= conf->reshape_progress) {
-+ previous = 1;
-+ } else {
-+ if (mddev->reshape_backwards
-+ ? logical_sector < conf->reshape_safe
-+ : logical_sector >= conf->reshape_safe) {
-+ spin_unlock_irq(&conf->device_lock);
-+ schedule();
-+ do_prepare = true;
-+ goto retry;
-+ }
-+ }
-+ spin_unlock_irq(&conf->device_lock);
-+ }
-+
-+ new_sector = raid5_compute_sector(conf, logical_sector,
-+ previous,
-+ &dd_idx, NULL);
-+ pr_debug("raid456: make_request, sector %llu logical %llu\n",
-+ (unsigned long long)new_sector,
-+ (unsigned long long)logical_sector);
-+
-+ sh = get_active_stripe(conf, new_sector, previous,
-+ (bi->bi_rw&RWA_MASK), 0);
-+ if (sh) {
-+ if (unlikely(previous)) {
-+ /* expansion might have moved on while waiting for a
-+ * stripe, so we must do the range check again.
-+ * Expansion could still move past after this
-+ * test, but as we are holding a reference to
-+ * 'sh', we know that if that happens,
-+ * STRIPE_EXPANDING will get set and the expansion
-+ * won't proceed until we finish with the stripe.
-+ */
-+ int must_retry = 0;
-+ spin_lock_irq(&conf->device_lock);
-+ if (mddev->reshape_backwards
-+ ? logical_sector >= conf->reshape_progress
-+ : logical_sector < conf->reshape_progress)
-+ /* mismatch, need to try again */
-+ must_retry = 1;
-+ spin_unlock_irq(&conf->device_lock);
-+ if (must_retry) {
-+ release_stripe(sh);
-+ schedule();
-+ do_prepare = true;
-+ goto retry;
-+ }
-+ }
-+ if (read_seqcount_retry(&conf->gen_lock, seq)) {
-+ /* Might have got the wrong stripe_head
-+ * by accident
-+ */
-+ release_stripe(sh);
-+ goto retry;
-+ }
-+
-+ if (rw == WRITE &&
-+ logical_sector >= mddev->suspend_lo &&
-+ logical_sector < mddev->suspend_hi) {
-+ release_stripe(sh);
-+ /* As the suspend_* range is controlled by
-+ * userspace, we want an interruptible
-+ * wait.
-+ */
-+ flush_signals(current);
-+ prepare_to_wait(&conf->wait_for_overlap,
-+ &w, TASK_INTERRUPTIBLE);
-+ if (logical_sector >= mddev->suspend_lo &&
-+ logical_sector < mddev->suspend_hi) {
-+ schedule();
-+ do_prepare = true;
-+ }
-+ goto retry;
-+ }
-+
-+ if (test_bit(STRIPE_EXPANDING, &sh->state) ||
-+ !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {
-+ /* Stripe is busy expanding or
-+ * add failed due to overlap. Flush everything
-+ * and wait a while
-+ */
-+ md_wakeup_thread(mddev->thread);
-+ release_stripe(sh);
-+ schedule();
-+ do_prepare = true;
-+ goto retry;
-+ }
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ clear_bit(STRIPE_DELAYED, &sh->state);
-+ if ((!sh->batch_head || sh == sh->batch_head) &&
-+ (bi->bi_rw & REQ_SYNC) &&
-+ !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-+ atomic_inc(&conf->preread_active_stripes);
-+ release_stripe_plug(mddev, sh);
-+ } else {
-+ /* cannot get stripe for read-ahead, just give-up */
-+ clear_bit(BIO_UPTODATE, &bi->bi_flags);
-+ break;
-+ }
-+ }
-+ finish_wait(&conf->wait_for_overlap, &w);
-+
-+ remaining = raid5_dec_bi_active_stripes(bi);
-+ if (remaining == 0) {
-+
-+ if ( rw == WRITE )
-+ md_write_end(mddev);
-+
-+ trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
-+ bi, 0);
-+ bio_endio(bi, 0);
-+ }
-+}
-+
-+static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
-+
-+static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
-+{
-+ /* reshaping is quite different to recovery/resync so it is
-+ * handled quite separately ... here.
-+ *
-+ * On each call to sync_request, we gather one chunk worth of
-+ * destination stripes and flag them as expanding.
-+ * Then we find all the source stripes and request reads.
-+ * As the reads complete, handle_stripe will copy the data
-+ * into the destination stripe and release that stripe.
-+ */
-+ struct r5conf *conf = mddev->private;
-+ struct stripe_head *sh;
-+ sector_t first_sector, last_sector;
-+ int raid_disks = conf->previous_raid_disks;
-+ int data_disks = raid_disks - conf->max_degraded;
-+ int new_data_disks = conf->raid_disks - conf->max_degraded;
-+ int i;
-+ int dd_idx;
-+ sector_t writepos, readpos, safepos;
-+ sector_t stripe_addr;
-+ int reshape_sectors;
-+ struct list_head stripes;
-+
-+ if (sector_nr == 0) {
-+ /* If restarting in the middle, skip the initial sectors */
-+ if (mddev->reshape_backwards &&
-+ conf->reshape_progress < raid5_size(mddev, 0, 0)) {
-+ sector_nr = raid5_size(mddev, 0, 0)
-+ - conf->reshape_progress;
-+ } else if (!mddev->reshape_backwards &&
-+ conf->reshape_progress > 0)
-+ sector_nr = conf->reshape_progress;
-+ sector_div(sector_nr, new_data_disks);
-+ if (sector_nr) {
-+ mddev->curr_resync_completed = sector_nr;
-+ sysfs_notify(&mddev->kobj, NULL, "sync_completed");
-+ *skipped = 1;
-+ return sector_nr;
-+ }
-+ }
-+
-+ /* We need to process a full chunk at a time.
-+ * If old and new chunk sizes differ, we need to process the
-+ * largest of these
-+ */
-+ if (mddev->new_chunk_sectors > mddev->chunk_sectors)
-+ reshape_sectors = mddev->new_chunk_sectors;
-+ else
-+ reshape_sectors = mddev->chunk_sectors;
-+
-+ /* We update the metadata at least every 10 seconds, or when
-+ * the data about to be copied would over-write the source of
-+ * the data at the front of the range. i.e. one new_stripe
-+ * along from reshape_progress new_maps to after where
-+ * reshape_safe old_maps to
-+ */
-+ writepos = conf->reshape_progress;
-+ sector_div(writepos, new_data_disks);
-+ readpos = conf->reshape_progress;
-+ sector_div(readpos, data_disks);
-+ safepos = conf->reshape_safe;
-+ sector_div(safepos, data_disks);
-+ if (mddev->reshape_backwards) {
-+ writepos -= min_t(sector_t, reshape_sectors, writepos);
-+ readpos += reshape_sectors;
-+ safepos += reshape_sectors;
-+ } else {
-+ writepos += reshape_sectors;
-+ readpos -= min_t(sector_t, reshape_sectors, readpos);
-+ safepos -= min_t(sector_t, reshape_sectors, safepos);
-+ }
-+
-+ /* Having calculated the 'writepos' possibly use it
-+ * to set 'stripe_addr' which is where we will write to.
-+ */
-+ if (mddev->reshape_backwards) {
-+ BUG_ON(conf->reshape_progress == 0);
-+ stripe_addr = writepos;
-+ BUG_ON((mddev->dev_sectors &
-+ ~((sector_t)reshape_sectors - 1))
-+ - reshape_sectors - stripe_addr
-+ != sector_nr);
-+ } else {
-+ BUG_ON(writepos != sector_nr + reshape_sectors);
-+ stripe_addr = sector_nr;
-+ }
-+
-+ /* 'writepos' is the most advanced device address we might write.
-+ * 'readpos' is the least advanced device address we might read.
-+ * 'safepos' is the least address recorded in the metadata as having
-+ * been reshaped.
-+ * If there is a min_offset_diff, these are adjusted either by
-+ * increasing the safepos/readpos if diff is negative, or
-+ * increasing writepos if diff is positive.
-+ * If 'readpos' is then behind 'writepos', there is no way that we can
-+ * ensure safety in the face of a crash - that must be done by userspace
-+ * making a backup of the data. So in that case there is no particular
-+ * rush to update metadata.
-+ * Otherwise if 'safepos' is behind 'writepos', then we really need to
-+ * update the metadata to advance 'safepos' to match 'readpos' so that
-+ * we can be safe in the event of a crash.
-+ * So we insist on updating metadata if safepos is behind writepos and
-+ * readpos is beyond writepos.
-+ * In any case, update the metadata every 10 seconds.
-+ * Maybe that number should be configurable, but I'm not sure it is
-+ * worth it.... maybe it could be a multiple of safemode_delay???
-+ */
-+ if (conf->min_offset_diff < 0) {
-+ safepos += -conf->min_offset_diff;
-+ readpos += -conf->min_offset_diff;
-+ } else
-+ writepos += conf->min_offset_diff;
-+
-+ if ((mddev->reshape_backwards
-+ ? (safepos > writepos && readpos < writepos)
-+ : (safepos < writepos && readpos > writepos)) ||
-+ time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
-+ /* Cannot proceed until we've updated the superblock... */
-+ wait_event(conf->wait_for_overlap,
-+ atomic_read(&conf->reshape_stripes)==0
-+ || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
-+ if (atomic_read(&conf->reshape_stripes) != 0)
-+ return 0;
-+ mddev->reshape_position = conf->reshape_progress;
-+ mddev->curr_resync_completed = sector_nr;
-+ conf->reshape_checkpoint = jiffies;
-+ set_bit(MD_CHANGE_DEVS, &mddev->flags);
-+ md_wakeup_thread(mddev->thread);
-+ wait_event(mddev->sb_wait, mddev->flags == 0 ||
-+ test_bit(MD_RECOVERY_INTR, &mddev->recovery));
-+ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
-+ return 0;
-+ spin_lock_irq(&conf->device_lock);
-+ conf->reshape_safe = mddev->reshape_position;
-+ spin_unlock_irq(&conf->device_lock);
-+ wake_up(&conf->wait_for_overlap);
-+ sysfs_notify(&mddev->kobj, NULL, "sync_completed");
-+ }
-+
-+ INIT_LIST_HEAD(&stripes);
-+ for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
-+ int j;
-+ int skipped_disk = 0;
-+ sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
-+ set_bit(STRIPE_EXPANDING, &sh->state);
-+ atomic_inc(&conf->reshape_stripes);
-+ /* If any of this stripe is beyond the end of the old
-+ * array, then we need to zero those blocks
-+ */
-+ for (j=sh->disks; j--;) {
-+ sector_t s;
-+ if (j == sh->pd_idx)
-+ continue;
-+ if (conf->level == 6 &&
-+ j == sh->qd_idx)
-+ continue;
-+ s = compute_blocknr(sh, j, 0);
-+ if (s < raid5_size(mddev, 0, 0)) {
-+ skipped_disk = 1;
-+ continue;
-+ }
-+ memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
-+ set_bit(R5_Expanded, &sh->dev[j].flags);
-+ set_bit(R5_UPTODATE, &sh->dev[j].flags);
-+ }
-+ if (!skipped_disk) {
-+ set_bit(STRIPE_EXPAND_READY, &sh->state);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ }
-+ list_add(&sh->lru, &stripes);
-+ }
-+ spin_lock_irq(&conf->device_lock);
-+ if (mddev->reshape_backwards)
-+ conf->reshape_progress -= reshape_sectors * new_data_disks;
-+ else
-+ conf->reshape_progress += reshape_sectors * new_data_disks;
-+ spin_unlock_irq(&conf->device_lock);
-+ /* Ok, those stripe are ready. We can start scheduling
-+ * reads on the source stripes.
-+ * The source stripes are determined by mapping the first and last
-+ * block on the destination stripes.
-+ */
-+ first_sector =
-+ raid5_compute_sector(conf, stripe_addr*(new_data_disks),
-+ 1, &dd_idx, NULL);
-+ last_sector =
-+ raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
-+ * new_data_disks - 1),
-+ 1, &dd_idx, NULL);
-+ if (last_sector >= mddev->dev_sectors)
-+ last_sector = mddev->dev_sectors - 1;
-+ while (first_sector <= last_sector) {
-+ sh = get_active_stripe(conf, first_sector, 1, 0, 1);
-+ set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+ release_stripe(sh);
-+ first_sector += STRIPE_SECTORS;
-+ }
-+ /* Now that the sources are clearly marked, we can release
-+ * the destination stripes
-+ */
-+ while (!list_empty(&stripes)) {
-+ sh = list_entry(stripes.next, struct stripe_head, lru);
-+ list_del_init(&sh->lru);
-+ release_stripe(sh);
-+ }
-+ /* If this takes us to the resync_max point where we have to pause,
-+ * then we need to write out the superblock.
-+ */
-+ sector_nr += reshape_sectors;
-+ if ((sector_nr - mddev->curr_resync_completed) * 2
-+ >= mddev->resync_max - mddev->curr_resync_completed) {
-+ /* Cannot proceed until we've updated the superblock... */
-+ wait_event(conf->wait_for_overlap,
-+ atomic_read(&conf->reshape_stripes) == 0
-+ || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
-+ if (atomic_read(&conf->reshape_stripes) != 0)
-+ goto ret;
-+ mddev->reshape_position = conf->reshape_progress;
-+ mddev->curr_resync_completed = sector_nr;
-+ conf->reshape_checkpoint = jiffies;
-+ set_bit(MD_CHANGE_DEVS, &mddev->flags);
-+ md_wakeup_thread(mddev->thread);
-+ wait_event(mddev->sb_wait,
-+ !test_bit(MD_CHANGE_DEVS, &mddev->flags)
-+ || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
-+ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
-+ goto ret;
-+ spin_lock_irq(&conf->device_lock);
-+ conf->reshape_safe = mddev->reshape_position;
-+ spin_unlock_irq(&conf->device_lock);
-+ wake_up(&conf->wait_for_overlap);
-+ sysfs_notify(&mddev->kobj, NULL, "sync_completed");
-+ }
-+ret:
-+ return reshape_sectors;
-+}
-+
-+static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
-+{
-+ struct r5conf *conf = mddev->private;
-+ struct stripe_head *sh;
-+ sector_t max_sector = mddev->dev_sectors;
-+ sector_t sync_blocks;
-+ int still_degraded = 0;
-+ int i;
-+
-+ if (sector_nr >= max_sector) {
-+ /* just being told to finish up .. nothing much to do */
-+
-+ if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
-+ end_reshape(conf);
-+ return 0;
-+ }
-+
-+ if (mddev->curr_resync < max_sector) /* aborted */
-+ bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
-+ &sync_blocks, 1);
-+ else /* completed sync */
-+ conf->fullsync = 0;
-+ bitmap_close_sync(mddev->bitmap);
-+
-+ return 0;
-+ }
-+
-+ /* Allow raid5_quiesce to complete */
-+ wait_event(conf->wait_for_overlap, conf->quiesce != 2);
-+
-+ if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
-+ return reshape_request(mddev, sector_nr, skipped);
-+
-+ /* No need to check resync_max as we never do more than one
-+ * stripe, and as resync_max will always be on a chunk boundary,
-+ * if the check in md_do_sync didn't fire, there is no chance
-+ * of overstepping resync_max here
-+ */
-+
-+ /* if there is too many failed drives and we are trying
-+ * to resync, then assert that we are finished, because there is
-+ * nothing we can do.
-+ */
-+ if (mddev->degraded >= conf->max_degraded &&
-+ test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
-+ sector_t rv = mddev->dev_sectors - sector_nr;
-+ *skipped = 1;
-+ return rv;
-+ }
-+ if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
-+ !conf->fullsync &&
-+ !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
-+ sync_blocks >= STRIPE_SECTORS) {
-+ /* we can skip this block, and probably more */
-+ sync_blocks /= STRIPE_SECTORS;
-+ *skipped = 1;
-+ return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
-+ }
-+
-+ bitmap_cond_end_sync(mddev->bitmap, sector_nr);
-+
-+ sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
-+ if (sh == NULL) {
-+ sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
-+ /* make sure we don't swamp the stripe cache if someone else
-+ * is trying to get access
-+ */
-+ schedule_timeout_uninterruptible(1);
-+ }
-+ /* Need to check if array will still be degraded after recovery/resync
-+ * Note in case of > 1 drive failures it's possible we're rebuilding
-+ * one drive while leaving another faulty drive in array.
-+ */
-+ rcu_read_lock();
-+ for (i = 0; i < conf->raid_disks; i++) {
-+ struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev);
-+
-+ if (rdev == NULL || test_bit(Faulty, &rdev->flags))
-+ still_degraded = 1;
-+ }
-+ rcu_read_unlock();
-+
-+ bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
-+
-+ set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
-+ set_bit(STRIPE_HANDLE, &sh->state);
-+
-+ release_stripe(sh);
-+
-+ return STRIPE_SECTORS;
-+}
-+
-+static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
-+{
-+ /* We may not be able to submit a whole bio at once as there
-+ * may not be enough stripe_heads available.
-+ * We cannot pre-allocate enough stripe_heads as we may need
-+ * more than exist in the cache (if we allow ever large chunks).
-+ * So we do one stripe head at a time and record in
-+ * ->bi_hw_segments how many have been done.
-+ *
-+ * We *know* that this entire raid_bio is in one chunk, so
-+ * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
-+ */
-+ struct stripe_head *sh;
-+ int dd_idx;
-+ sector_t sector, logical_sector, last_sector;
-+ int scnt = 0;
-+ int remaining;
-+ int handled = 0;
-+
-+ logical_sector = raid_bio->bi_iter.bi_sector &
-+ ~((sector_t)STRIPE_SECTORS-1);
-+ sector = raid5_compute_sector(conf, logical_sector,
-+ 0, &dd_idx, NULL);
-+ last_sector = bio_end_sector(raid_bio);
-+
-+ for (; logical_sector < last_sector;
-+ logical_sector += STRIPE_SECTORS,
-+ sector += STRIPE_SECTORS,
-+ scnt++) {
-+
-+ if (scnt < raid5_bi_processed_stripes(raid_bio))
-+ /* already done this stripe */
-+ continue;
-+
-+ sh = get_active_stripe(conf, sector, 0, 1, 1);
-+
-+ if (!sh) {
-+ /* failed to get a stripe - must wait */
-+ raid5_set_bi_processed_stripes(raid_bio, scnt);
-+ conf->retry_read_aligned = raid_bio;
-+ return handled;
-+ }
-+
-+ if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
-+ release_stripe(sh);
-+ raid5_set_bi_processed_stripes(raid_bio, scnt);
-+ conf->retry_read_aligned = raid_bio;
-+ return handled;
-+ }
-+
-+ set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
-+ handle_stripe(sh);
-+ release_stripe(sh);
-+ handled++;
-+ }
-+ remaining = raid5_dec_bi_active_stripes(raid_bio);
-+ if (remaining == 0) {
-+ trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev),
-+ raid_bio, 0);
-+ bio_endio(raid_bio, 0);
-+ }
-+ if (atomic_dec_and_test(&conf->active_aligned_reads))
-+ wake_up(&conf->wait_for_stripe);
-+ return handled;
-+}
-+
-+static int handle_active_stripes(struct r5conf *conf, int group,
-+ struct r5worker *worker,
-+ struct list_head *temp_inactive_list)
-+{
-+ struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
-+ int i, batch_size = 0, hash;
-+ bool release_inactive = false;
-+
-+ while (batch_size < MAX_STRIPE_BATCH &&
-+ (sh = __get_priority_stripe(conf, group)) != NULL)
-+ batch[batch_size++] = sh;
-+
-+ if (batch_size == 0) {
-+ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
-+ if (!list_empty(temp_inactive_list + i))
-+ break;
-+ if (i == NR_STRIPE_HASH_LOCKS)
-+ return batch_size;
-+ release_inactive = true;
-+ }
-+ spin_unlock_irq(&conf->device_lock);
-+
-+ release_inactive_stripe_list(conf, temp_inactive_list,
-+ NR_STRIPE_HASH_LOCKS);
-+
-+ if (release_inactive) {
-+ spin_lock_irq(&conf->device_lock);
-+ return 0;
-+ }
-+
-+ for (i = 0; i < batch_size; i++)
-+ handle_stripe(batch[i]);
-+
-+ cond_resched();
-+
-+ spin_lock_irq(&conf->device_lock);
-+ for (i = 0; i < batch_size; i++) {
-+ hash = batch[i]->hash_lock_index;
-+ __release_stripe(conf, batch[i], &temp_inactive_list[hash]);
-+ }
-+ return batch_size;
-+}
-+
-+static void raid5_do_work(struct work_struct *work)
-+{
-+ struct r5worker *worker = container_of(work, struct r5worker, work);
-+ struct r5worker_group *group = worker->group;
-+ struct r5conf *conf = group->conf;
-+ int group_id = group - conf->worker_groups;
-+ int handled;
-+ struct blk_plug plug;
-+
-+ pr_debug("+++ raid5worker active\n");
-+
-+ blk_start_plug(&plug);
-+ handled = 0;
-+ spin_lock_irq(&conf->device_lock);
-+ while (1) {
-+ int batch_size, released;
-+
-+ released = release_stripe_list(conf, worker->temp_inactive_list);
-+
-+ batch_size = handle_active_stripes(conf, group_id, worker,
-+ worker->temp_inactive_list);
-+ worker->working = false;
-+ if (!batch_size && !released)
-+ break;
-+ handled += batch_size;
-+ }
-+ pr_debug("%d stripes handled\n", handled);
-+
-+ spin_unlock_irq(&conf->device_lock);
-+ blk_finish_plug(&plug);
-+
-+ pr_debug("--- raid5worker inactive\n");
-+}
-+
-+/*
-+ * This is our raid5 kernel thread.
-+ *
-+ * We scan the hash table for stripes which can be handled now.
-+ * During the scan, completed stripes are saved for us by the interrupt
-+ * handler, so that they will not have to wait for our next wakeup.
-+ */
-+static void raid5d(struct md_thread *thread)
-+{
-+ struct mddev *mddev = thread->mddev;
-+ struct r5conf *conf = mddev->private;
-+ int handled;
-+ struct blk_plug plug;
-+
-+ pr_debug("+++ raid5d active\n");
-+
-+ md_check_recovery(mddev);
-+
-+ blk_start_plug(&plug);
-+ handled = 0;
-+ spin_lock_irq(&conf->device_lock);
-+ while (1) {
-+ struct bio *bio;
-+ int batch_size, released;
-+
-+ released = release_stripe_list(conf, conf->temp_inactive_list);
-+ if (released)
-+ clear_bit(R5_DID_ALLOC, &conf->cache_state);
-+
-+ if (
-+ !list_empty(&conf->bitmap_list)) {
-+ /* Now is a good time to flush some bitmap updates */
-+ conf->seq_flush++;
-+ spin_unlock_irq(&conf->device_lock);
-+ bitmap_unplug(mddev->bitmap);
-+ spin_lock_irq(&conf->device_lock);
-+ conf->seq_write = conf->seq_flush;
-+ activate_bit_delay(conf, conf->temp_inactive_list);
-+ }
-+ raid5_activate_delayed(conf);
-+
-+ while ((bio = remove_bio_from_retry(conf))) {
-+ int ok;
-+ spin_unlock_irq(&conf->device_lock);
-+ ok = retry_aligned_read(conf, bio);
-+ spin_lock_irq(&conf->device_lock);
-+ if (!ok)
-+ break;
-+ handled++;
-+ }
-+
-+ batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
-+ conf->temp_inactive_list);
-+ if (!batch_size && !released)
-+ break;
-+ handled += batch_size;
-+
-+ if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
-+ spin_unlock_irq(&conf->device_lock);
-+ md_check_recovery(mddev);
-+ spin_lock_irq(&conf->device_lock);
-+ }
-+ }
-+ pr_debug("%d stripes handled\n", handled);
-+
-+ spin_unlock_irq(&conf->device_lock);
-+ if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) &&
-+ mutex_trylock(&conf->cache_size_mutex)) {
-+ grow_one_stripe(conf, __GFP_NOWARN);
-+ /* Set flag even if allocation failed. This helps
-+ * slow down allocation requests when mem is short
-+ */
-+ set_bit(R5_DID_ALLOC, &conf->cache_state);
-+ mutex_unlock(&conf->cache_size_mutex);
-+ }
-+
-+ async_tx_issue_pending_all();
-+ blk_finish_plug(&plug);
-+
-+ pr_debug("--- raid5d inactive\n");
-+}
-+
-+static ssize_t
-+raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
-+{
-+ struct r5conf *conf;
-+ int ret = 0;
-+ spin_lock(&mddev->lock);
-+ conf = mddev->private;
-+ if (conf)
-+ ret = sprintf(page, "%d\n", conf->min_nr_stripes);
-+ spin_unlock(&mddev->lock);
-+ return ret;
-+}
-+
-+int
-+raid5_set_cache_size(struct mddev *mddev, int size)
-+{
-+ struct r5conf *conf = mddev->private;
-+ int err;
-+
-+ if (size <= 16 || size > 32768)
-+ return -EINVAL;
-+
-+ conf->min_nr_stripes = size;
-+ mutex_lock(&conf->cache_size_mutex);
-+ while (size < conf->max_nr_stripes &&
-+ drop_one_stripe(conf))
-+ ;
-+ mutex_unlock(&conf->cache_size_mutex);
-+
-+
-+ err = md_allow_write(mddev);
-+ if (err)
-+ return err;
-+
-+ mutex_lock(&conf->cache_size_mutex);
-+ while (size > conf->max_nr_stripes)
-+ if (!grow_one_stripe(conf, GFP_KERNEL))
-+ break;
-+ mutex_unlock(&conf->cache_size_mutex);
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(raid5_set_cache_size);
-+
-+static ssize_t
-+raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
-+{
-+ struct r5conf *conf;
-+ unsigned long new;
-+ int err;
-+
-+ if (len >= PAGE_SIZE)
-+ return -EINVAL;
-+ if (kstrtoul(page, 10, &new))
-+ return -EINVAL;
-+ err = mddev_lock(mddev);
-+ if (err)
-+ return err;
-+ conf = mddev->private;
-+ if (!conf)
-+ err = -ENODEV;
-+ else
-+ err = raid5_set_cache_size(mddev, new);
-+ mddev_unlock(mddev);
-+
-+ return err ?: len;
-+}
-+
-+static struct md_sysfs_entry
-+raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
-+ raid5_show_stripe_cache_size,
-+ raid5_store_stripe_cache_size);
-+
-+static ssize_t
-+raid5_show_rmw_level(struct mddev *mddev, char *page)
-+{
-+ struct r5conf *conf = mddev->private;
-+ if (conf)
-+ return sprintf(page, "%d\n", conf->rmw_level);
-+ else
-+ return 0;
-+}
-+
-+static ssize_t
-+raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len)
-+{
-+ struct r5conf *conf = mddev->private;
-+ unsigned long new;
-+
-+ if (!conf)
-+ return -ENODEV;
-+
-+ if (len >= PAGE_SIZE)
-+ return -EINVAL;
-+
-+ if (kstrtoul(page, 10, &new))
-+ return -EINVAL;
-+
-+ if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)
-+ return -EINVAL;
-+
-+ if (new != PARITY_DISABLE_RMW &&
-+ new != PARITY_ENABLE_RMW &&
-+ new != PARITY_PREFER_RMW)
-+ return -EINVAL;
-+
-+ conf->rmw_level = new;
-+ return len;
-+}
-+
-+static struct md_sysfs_entry
-+raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR,
-+ raid5_show_rmw_level,
-+ raid5_store_rmw_level);
-+
-+
-+static ssize_t
-+raid5_show_preread_threshold(struct mddev *mddev, char *page)
-+{
-+ struct r5conf *conf;
-+ int ret = 0;
-+ spin_lock(&mddev->lock);
-+ conf = mddev->private;
-+ if (conf)
-+ ret = sprintf(page, "%d\n", conf->bypass_threshold);
-+ spin_unlock(&mddev->lock);
-+ return ret;
-+}
-+
-+static ssize_t
-+raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
-+{
-+ struct r5conf *conf;
-+ unsigned long new;
-+ int err;
-+
-+ if (len >= PAGE_SIZE)
-+ return -EINVAL;
-+ if (kstrtoul(page, 10, &new))
-+ return -EINVAL;
-+
-+ err = mddev_lock(mddev);
-+ if (err)
-+ return err;
-+ conf = mddev->private;
-+ if (!conf)
-+ err = -ENODEV;
-+ else if (new > conf->min_nr_stripes)
-+ err = -EINVAL;
-+ else
-+ conf->bypass_threshold = new;
-+ mddev_unlock(mddev);
-+ return err ?: len;
-+}
-+
-+static struct md_sysfs_entry
-+raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
-+ S_IRUGO | S_IWUSR,
-+ raid5_show_preread_threshold,
-+ raid5_store_preread_threshold);
-+
-+static ssize_t
-+raid5_show_skip_copy(struct mddev *mddev, char *page)
-+{
-+ struct r5conf *conf;
-+ int ret = 0;
-+ spin_lock(&mddev->lock);
-+ conf = mddev->private;
-+ if (conf)
-+ ret = sprintf(page, "%d\n", conf->skip_copy);
-+ spin_unlock(&mddev->lock);
-+ return ret;
-+}
-+
-+static ssize_t
-+raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)
-+{
-+ struct r5conf *conf;
-+ unsigned long new;
-+ int err;
-+
-+ if (len >= PAGE_SIZE)
-+ return -EINVAL;
-+ if (kstrtoul(page, 10, &new))
-+ return -EINVAL;
-+ new = !!new;
-+
-+ err = mddev_lock(mddev);
-+ if (err)
-+ return err;
-+ conf = mddev->private;
-+ if (!conf)
-+ err = -ENODEV;
-+ else if (new != conf->skip_copy) {
-+ mddev_suspend(mddev);
-+ conf->skip_copy = new;
-+ if (new)
-+ mddev->queue->backing_dev_info.capabilities |=
-+ BDI_CAP_STABLE_WRITES;
-+ else
-+ mddev->queue->backing_dev_info.capabilities &=
-+ ~BDI_CAP_STABLE_WRITES;
-+ mddev_resume(mddev);
-+ }
-+ mddev_unlock(mddev);
-+ return err ?: len;
-+}
-+
-+static struct md_sysfs_entry
-+raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,
-+ raid5_show_skip_copy,
-+ raid5_store_skip_copy);
-+
-+static ssize_t
-+stripe_cache_active_show(struct mddev *mddev, char *page)
-+{
-+ struct r5conf *conf = mddev->private;
-+ if (conf)
-+ return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
-+ else
-+ return 0;
-+}
-+
-+static struct md_sysfs_entry
-+raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
-+
-+static ssize_t
-+raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
-+{
-+ struct r5conf *conf;
-+ int ret = 0;
-+ spin_lock(&mddev->lock);
-+ conf = mddev->private;
-+ if (conf)
-+ ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);
-+ spin_unlock(&mddev->lock);
-+ return ret;
-+}
-+
-+static int alloc_thread_groups(struct r5conf *conf, int cnt,
-+ int *group_cnt,
-+ int *worker_cnt_per_group,
-+ struct r5worker_group **worker_groups);
-+static ssize_t
-+raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
-+{
-+ struct r5conf *conf;
-+ unsigned long new;
-+ int err;
-+ struct r5worker_group *new_groups, *old_groups;
-+ int group_cnt, worker_cnt_per_group;
-+
-+ if (len >= PAGE_SIZE)
-+ return -EINVAL;
-+ if (kstrtoul(page, 10, &new))
-+ return -EINVAL;
-+
-+ err = mddev_lock(mddev);
-+ if (err)
-+ return err;
-+ conf = mddev->private;
-+ if (!conf)
-+ err = -ENODEV;
-+ else if (new != conf->worker_cnt_per_group) {
-+ mddev_suspend(mddev);
-+
-+ old_groups = conf->worker_groups;
-+ if (old_groups)
-+ flush_workqueue(raid5_wq);
-+
-+ err = alloc_thread_groups(conf, new,
-+ &group_cnt, &worker_cnt_per_group,
-+ &new_groups);
-+ if (!err) {
-+ spin_lock_irq(&conf->device_lock);
-+ conf->group_cnt = group_cnt;
-+ conf->worker_cnt_per_group = worker_cnt_per_group;
-+ conf->worker_groups = new_groups;
-+ spin_unlock_irq(&conf->device_lock);
-+
-+ if (old_groups)
-+ kfree(old_groups[0].workers);
-+ kfree(old_groups);
-+ }
-+ mddev_resume(mddev);
-+ }
-+ mddev_unlock(mddev);
-+
-+ return err ?: len;
-+}
-+
-+static struct md_sysfs_entry
-+raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
-+ raid5_show_group_thread_cnt,
-+ raid5_store_group_thread_cnt);
-+
-+static struct attribute *raid5_attrs[] = {
-+ &raid5_stripecache_size.attr,
-+ &raid5_stripecache_active.attr,
-+ &raid5_preread_bypass_threshold.attr,
-+ &raid5_group_thread_cnt.attr,
-+ &raid5_skip_copy.attr,
-+ &raid5_rmw_level.attr,
-+ NULL,
-+};
-+static struct attribute_group raid5_attrs_group = {
-+ .name = NULL,
-+ .attrs = raid5_attrs,
-+};
-+
-+static int alloc_thread_groups(struct r5conf *conf, int cnt,
-+ int *group_cnt,
-+ int *worker_cnt_per_group,
-+ struct r5worker_group **worker_groups)
-+{
-+ int i, j, k;
-+ ssize_t size;
-+ struct r5worker *workers;
-+
-+ *worker_cnt_per_group = cnt;
-+ if (cnt == 0) {
-+ *group_cnt = 0;
-+ *worker_groups = NULL;
-+ return 0;
-+ }
-+ *group_cnt = num_possible_nodes();
-+ size = sizeof(struct r5worker) * cnt;
-+ workers = kzalloc(size * *group_cnt, GFP_NOIO);
-+ *worker_groups = kzalloc(sizeof(struct r5worker_group) *
-+ *group_cnt, GFP_NOIO);
-+ if (!*worker_groups || !workers) {
-+ kfree(workers);
-+ kfree(*worker_groups);
-+ return -ENOMEM;
-+ }
-+
-+ for (i = 0; i < *group_cnt; i++) {
-+ struct r5worker_group *group;
-+
-+ group = &(*worker_groups)[i];
-+ INIT_LIST_HEAD(&group->handle_list);
-+ group->conf = conf;
-+ group->workers = workers + i * cnt;
-+
-+ for (j = 0; j < cnt; j++) {
-+ struct r5worker *worker = group->workers + j;
-+ worker->group = group;
-+ INIT_WORK(&worker->work, raid5_do_work);
-+
-+ for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
-+ INIT_LIST_HEAD(worker->temp_inactive_list + k);
-+ }
-+ }
-+
-+ return 0;
-+}
-+
-+static void free_thread_groups(struct r5conf *conf)
-+{
-+ if (conf->worker_groups)
-+ kfree(conf->worker_groups[0].workers);
-+ kfree(conf->worker_groups);
-+ conf->worker_groups = NULL;
-+}
-+
-+static sector_t
-+raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
-+{
-+ struct r5conf *conf = mddev->private;
-+
-+ if (!sectors)
-+ sectors = mddev->dev_sectors;
-+ if (!raid_disks)
-+ /* size is defined by the smallest of previous and new size */
-+ raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
-+
-+ sectors &= ~((sector_t)mddev->chunk_sectors - 1);
-+ sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
-+ return sectors * (raid_disks - conf->max_degraded);
-+}
-+
-+static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
-+{
-+ safe_put_page(percpu->spare_page);
-+ if (percpu->scribble)
-+ flex_array_free(percpu->scribble);
-+ percpu->spare_page = NULL;
-+ percpu->scribble = NULL;
-+}
-+
-+static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
-+{
-+ if (conf->level == 6 && !percpu->spare_page)
-+ percpu->spare_page = alloc_page(GFP_KERNEL);
-+ if (!percpu->scribble)
-+ percpu->scribble = scribble_alloc(max(conf->raid_disks,
-+ conf->previous_raid_disks),
-+ max(conf->chunk_sectors,
-+ conf->prev_chunk_sectors)
-+ / STRIPE_SECTORS,
-+ GFP_KERNEL);
-+
-+ if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) {
-+ free_scratch_buffer(conf, percpu);
-+ return -ENOMEM;
-+ }
-+
-+ return 0;
-+}
-+
-+static void raid5_free_percpu(struct r5conf *conf)
-+{
-+ unsigned long cpu;
-+
-+ if (!conf->percpu)
-+ return;
-+
-+#ifdef CONFIG_HOTPLUG_CPU
-+ unregister_cpu_notifier(&conf->cpu_notify);
-+#endif
-+
-+ get_online_cpus();
-+ for_each_possible_cpu(cpu)
-+ free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
-+ put_online_cpus();
-+
-+ free_percpu(conf->percpu);
-+}
-+
-+static void free_conf(struct r5conf *conf)
-+{
-+ if (conf->shrinker.seeks)
-+ unregister_shrinker(&conf->shrinker);
-+ free_thread_groups(conf);
-+ shrink_stripes(conf);
-+ raid5_free_percpu(conf);
-+ kfree(conf->disks);
-+ kfree(conf->stripe_hashtbl);
-+ kfree(conf);
-+}
-+
-+#ifdef CONFIG_HOTPLUG_CPU
-+static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
-+ void *hcpu)
-+{
-+ struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
-+ long cpu = (long)hcpu;
-+ struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
-+
-+ switch (action) {
-+ case CPU_UP_PREPARE:
-+ case CPU_UP_PREPARE_FROZEN:
-+ if (alloc_scratch_buffer(conf, percpu)) {
-+ pr_err("%s: failed memory allocation for cpu%ld\n",
-+ __func__, cpu);
-+ return notifier_from_errno(-ENOMEM);
-+ }
-+ break;
-+ case CPU_DEAD:
-+ case CPU_DEAD_FROZEN:
-+ free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
-+ break;
-+ default:
-+ break;
-+ }
-+ return NOTIFY_OK;
-+}
-+#endif
-+
-+static int raid5_alloc_percpu(struct r5conf *conf)
-+{
-+ unsigned long cpu;
-+ int err = 0;
-+
-+ conf->percpu = alloc_percpu(struct raid5_percpu);
-+ if (!conf->percpu)
-+ return -ENOMEM;
-+
-+#ifdef CONFIG_HOTPLUG_CPU
-+ conf->cpu_notify.notifier_call = raid456_cpu_notify;
-+ conf->cpu_notify.priority = 0;
-+ err = register_cpu_notifier(&conf->cpu_notify);
-+ if (err)
-+ return err;
-+#endif
-+
-+ get_online_cpus();
-+ for_each_present_cpu(cpu) {
-+ err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
-+ if (err) {
-+ pr_err("%s: failed memory allocation for cpu%ld\n",
-+ __func__, cpu);
-+ break;
-+ }
-+ }
-+ put_online_cpus();
-+
-+ return err;
-+}
-+
-+static unsigned long raid5_cache_scan(struct shrinker *shrink,
-+ struct shrink_control *sc)
-+{
-+ struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
-+ unsigned long ret = SHRINK_STOP;
-+
-+ if (mutex_trylock(&conf->cache_size_mutex)) {
-+ ret= 0;
-+ while (ret < sc->nr_to_scan &&
-+ conf->max_nr_stripes > conf->min_nr_stripes) {
-+ if (drop_one_stripe(conf) == 0) {
-+ ret = SHRINK_STOP;
-+ break;
-+ }
-+ ret++;
-+ }
-+ mutex_unlock(&conf->cache_size_mutex);
-+ }
-+ return ret;
-+}
-+
-+static unsigned long raid5_cache_count(struct shrinker *shrink,
-+ struct shrink_control *sc)
-+{
-+ struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
-+
-+ if (conf->max_nr_stripes < conf->min_nr_stripes)
-+ /* unlikely, but not impossible */
-+ return 0;
-+ return conf->max_nr_stripes - conf->min_nr_stripes;
-+}
-+
-+static struct r5conf *setup_conf(struct mddev *mddev)
-+{
-+ struct r5conf *conf;
-+ int raid_disk, memory, max_disks;
-+ struct md_rdev *rdev;
-+ struct disk_info *disk;
-+ char pers_name[6];
-+ int i;
-+ int group_cnt, worker_cnt_per_group;
-+ struct r5worker_group *new_group;
-+
-+ if (mddev->new_level != 5
-+ && mddev->new_level != 4
-+ && mddev->new_level != 6) {
-+ printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
-+ mdname(mddev), mddev->new_level);
-+ return ERR_PTR(-EIO);
-+ }
-+ if ((mddev->new_level == 5
-+ && !algorithm_valid_raid5(mddev->new_layout)) ||
-+ (mddev->new_level == 6
-+ && !algorithm_valid_raid6(mddev->new_layout))) {
-+ printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
-+ mdname(mddev), mddev->new_layout);
-+ return ERR_PTR(-EIO);
-+ }
-+ if (mddev->new_level == 6 && mddev->raid_disks < 4) {
-+ printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
-+ mdname(mddev), mddev->raid_disks);
-+ return ERR_PTR(-EINVAL);
-+ }
-+
-+ if (!mddev->new_chunk_sectors ||
-+ (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
-+ !is_power_of_2(mddev->new_chunk_sectors)) {
-+ printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
-+ mdname(mddev), mddev->new_chunk_sectors << 9);
-+ return ERR_PTR(-EINVAL);
-+ }
-+
-+ conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
-+ if (conf == NULL)
-+ goto abort;
-+ /* Don't enable multi-threading by default*/
-+ if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
-+ &new_group)) {
-+ conf->group_cnt = group_cnt;
-+ conf->worker_cnt_per_group = worker_cnt_per_group;
-+ conf->worker_groups = new_group;
-+ } else
-+ goto abort;
-+ spin_lock_init(&conf->device_lock);
-+ seqcount_init(&conf->gen_lock);
-+ mutex_init(&conf->cache_size_mutex);
-+ init_waitqueue_head(&conf->wait_for_stripe);
-+ init_waitqueue_head(&conf->wait_for_overlap);
-+ INIT_LIST_HEAD(&conf->handle_list);
-+ INIT_LIST_HEAD(&conf->hold_list);
-+ INIT_LIST_HEAD(&conf->delayed_list);
-+ INIT_LIST_HEAD(&conf->bitmap_list);
-+ init_llist_head(&conf->released_stripes);
-+ atomic_set(&conf->active_stripes, 0);
-+ atomic_set(&conf->preread_active_stripes, 0);
-+ atomic_set(&conf->active_aligned_reads, 0);
-+ conf->bypass_threshold = BYPASS_THRESHOLD;
-+ conf->recovery_disabled = mddev->recovery_disabled - 1;
-+
-+ conf->raid_disks = mddev->raid_disks;
-+ if (mddev->reshape_position == MaxSector)
-+ conf->previous_raid_disks = mddev->raid_disks;
-+ else
-+ conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
-+ max_disks = max(conf->raid_disks, conf->previous_raid_disks);
-+
-+ conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
-+ GFP_KERNEL);
-+ if (!conf->disks)
-+ goto abort;
-+
-+ conf->mddev = mddev;
-+
-+ if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
-+ goto abort;
-+
-+ /* We init hash_locks[0] separately to that it can be used
-+ * as the reference lock in the spin_lock_nest_lock() call
-+ * in lock_all_device_hash_locks_irq in order to convince
-+ * lockdep that we know what we are doing.
-+ */
-+ spin_lock_init(conf->hash_locks);
-+ for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
-+ spin_lock_init(conf->hash_locks + i);
-+
-+ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
-+ INIT_LIST_HEAD(conf->inactive_list + i);
-+
-+ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
-+ INIT_LIST_HEAD(conf->temp_inactive_list + i);
-+
-+ conf->level = mddev->new_level;
-+ conf->chunk_sectors = mddev->new_chunk_sectors;
-+ if (raid5_alloc_percpu(conf) != 0)
-+ goto abort;
-+
-+ pr_debug("raid456: run(%s) called.\n", mdname(mddev));
-+
-+ rdev_for_each(rdev, mddev) {
-+ raid_disk = rdev->raid_disk;
-+ if (raid_disk >= max_disks
-+ || raid_disk < 0)
-+ continue;
-+ disk = conf->disks + raid_disk;
-+
-+ if (test_bit(Replacement, &rdev->flags)) {
-+ if (disk->replacement)
-+ goto abort;
-+ disk->replacement = rdev;
-+ } else {
-+ if (disk->rdev)
-+ goto abort;
-+ disk->rdev = rdev;
-+ }
-+
-+ if (test_bit(In_sync, &rdev->flags)) {
-+ char b[BDEVNAME_SIZE];
-+ printk(KERN_INFO "md/raid:%s: device %s operational as raid"
-+ " disk %d\n",
-+ mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
-+ } else if (rdev->saved_raid_disk != raid_disk)
-+ /* Cannot rely on bitmap to complete recovery */
-+ conf->fullsync = 1;
-+ }
-+
-+ conf->level = mddev->new_level;
-+ if (conf->level == 6) {
-+ conf->max_degraded = 2;
-+ if (raid6_call.xor_syndrome)
-+ conf->rmw_level = PARITY_ENABLE_RMW;
-+ else
-+ conf->rmw_level = PARITY_DISABLE_RMW;
-+ } else {
-+ conf->max_degraded = 1;
-+ conf->rmw_level = PARITY_ENABLE_RMW;
-+ }
-+ conf->algorithm = mddev->new_layout;
-+ conf->reshape_progress = mddev->reshape_position;
-+ if (conf->reshape_progress != MaxSector) {
-+ conf->prev_chunk_sectors = mddev->chunk_sectors;
-+ conf->prev_algo = mddev->layout;
-+ }
-+
-+ conf->min_nr_stripes = NR_STRIPES;
-+ memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +
-+ max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
-+ atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
-+ if (grow_stripes(conf, conf->min_nr_stripes)) {
-+ printk(KERN_ERR
-+ "md/raid:%s: couldn't allocate %dkB for buffers\n",
-+ mdname(mddev), memory);
-+ goto abort;
-+ } else
-+ printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
-+ mdname(mddev), memory);
-+ /*
-+ * Losing a stripe head costs more than the time to refill it,
-+ * it reduces the queue depth and so can hurt throughput.
-+ * So set it rather large, scaled by number of devices.
-+ */
-+ conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4;
-+ conf->shrinker.scan_objects = raid5_cache_scan;
-+ conf->shrinker.count_objects = raid5_cache_count;
-+ conf->shrinker.batch = 128;
-+ conf->shrinker.flags = 0;
-+ register_shrinker(&conf->shrinker);
-+
-+ sprintf(pers_name, "raid%d", mddev->new_level);
-+ conf->thread = md_register_thread(raid5d, mddev, pers_name);
-+ if (!conf->thread) {
-+ printk(KERN_ERR
-+ "md/raid:%s: couldn't allocate thread.\n",
-+ mdname(mddev));
-+ goto abort;
-+ }
-+
-+ return conf;
-+
-+ abort:
-+ if (conf) {
-+ free_conf(conf);
-+ return ERR_PTR(-EIO);
-+ } else
-+ return ERR_PTR(-ENOMEM);
-+}
-+
-+static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
-+{
-+ switch (algo) {
-+ case ALGORITHM_PARITY_0:
-+ if (raid_disk < max_degraded)
-+ return 1;
-+ break;
-+ case ALGORITHM_PARITY_N:
-+ if (raid_disk >= raid_disks - max_degraded)
-+ return 1;
-+ break;
-+ case ALGORITHM_PARITY_0_6:
-+ if (raid_disk == 0 ||
-+ raid_disk == raid_disks - 1)
-+ return 1;
-+ break;
-+ case ALGORITHM_LEFT_ASYMMETRIC_6:
-+ case ALGORITHM_RIGHT_ASYMMETRIC_6:
-+ case ALGORITHM_LEFT_SYMMETRIC_6:
-+ case ALGORITHM_RIGHT_SYMMETRIC_6:
-+ if (raid_disk == raid_disks - 1)
-+ return 1;
-+ }
-+ return 0;
-+}
-+
-+static int run(struct mddev *mddev)
-+{
-+ struct r5conf *conf;
-+ int working_disks = 0;
-+ int dirty_parity_disks = 0;
-+ struct md_rdev *rdev;
-+ sector_t reshape_offset = 0;
-+ int i;
-+ long long min_offset_diff = 0;
-+ int first = 1;
-+
-+ if (mddev->recovery_cp != MaxSector)
-+ printk(KERN_NOTICE "md/raid:%s: not clean"
-+ " -- starting background reconstruction\n",
-+ mdname(mddev));
-+
-+ rdev_for_each(rdev, mddev) {
-+ long long diff;
-+ if (rdev->raid_disk < 0)
-+ continue;
-+ diff = (rdev->new_data_offset - rdev->data_offset);
-+ if (first) {
-+ min_offset_diff = diff;
-+ first = 0;
-+ } else if (mddev->reshape_backwards &&
-+ diff < min_offset_diff)
-+ min_offset_diff = diff;
-+ else if (!mddev->reshape_backwards &&
-+ diff > min_offset_diff)
-+ min_offset_diff = diff;
-+ }
-+
-+ if (mddev->reshape_position != MaxSector) {
-+ /* Check that we can continue the reshape.
-+ * Difficulties arise if the stripe we would write to
-+ * next is at or after the stripe we would read from next.
-+ * For a reshape that changes the number of devices, this
-+ * is only possible for a very short time, and mdadm makes
-+ * sure that time appears to have past before assembling
-+ * the array. So we fail if that time hasn't passed.
-+ * For a reshape that keeps the number of devices the same
-+ * mdadm must be monitoring the reshape can keeping the
-+ * critical areas read-only and backed up. It will start
-+ * the array in read-only mode, so we check for that.
-+ */
-+ sector_t here_new, here_old;
-+ int old_disks;
-+ int max_degraded = (mddev->level == 6 ? 2 : 1);
-+
-+ if (mddev->new_level != mddev->level) {
-+ printk(KERN_ERR "md/raid:%s: unsupported reshape "
-+ "required - aborting.\n",
-+ mdname(mddev));
-+ return -EINVAL;
-+ }
-+ old_disks = mddev->raid_disks - mddev->delta_disks;
-+ /* reshape_position must be on a new-stripe boundary, and one
-+ * further up in new geometry must map after here in old
-+ * geometry.
-+ */
-+ here_new = mddev->reshape_position;
-+ if (sector_div(here_new, mddev->new_chunk_sectors *
-+ (mddev->raid_disks - max_degraded))) {
-+ printk(KERN_ERR "md/raid:%s: reshape_position not "
-+ "on a stripe boundary\n", mdname(mddev));
-+ return -EINVAL;
-+ }
-+ reshape_offset = here_new * mddev->new_chunk_sectors;
-+ /* here_new is the stripe we will write to */
-+ here_old = mddev->reshape_position;
-+ sector_div(here_old, mddev->chunk_sectors *
-+ (old_disks-max_degraded));
-+ /* here_old is the first stripe that we might need to read
-+ * from */
-+ if (mddev->delta_disks == 0) {
-+ if ((here_new * mddev->new_chunk_sectors !=
-+ here_old * mddev->chunk_sectors)) {
-+ printk(KERN_ERR "md/raid:%s: reshape position is"
-+ " confused - aborting\n", mdname(mddev));
-+ return -EINVAL;
-+ }
-+ /* We cannot be sure it is safe to start an in-place
-+ * reshape. It is only safe if user-space is monitoring
-+ * and taking constant backups.
-+ * mdadm always starts a situation like this in
-+ * readonly mode so it can take control before
-+ * allowing any writes. So just check for that.
-+ */
-+ if (abs(min_offset_diff) >= mddev->chunk_sectors &&
-+ abs(min_offset_diff) >= mddev->new_chunk_sectors)
-+ /* not really in-place - so OK */;
-+ else if (mddev->ro == 0) {
-+ printk(KERN_ERR "md/raid:%s: in-place reshape "
-+ "must be started in read-only mode "
-+ "- aborting\n",
-+ mdname(mddev));
-+ return -EINVAL;
-+ }
-+ } else if (mddev->reshape_backwards
-+ ? (here_new * mddev->new_chunk_sectors + min_offset_diff <=
-+ here_old * mddev->chunk_sectors)
-+ : (here_new * mddev->new_chunk_sectors >=
-+ here_old * mddev->chunk_sectors + (-min_offset_diff))) {
-+ /* Reading from the same stripe as writing to - bad */
-+ printk(KERN_ERR "md/raid:%s: reshape_position too early for "
-+ "auto-recovery - aborting.\n",
-+ mdname(mddev));
-+ return -EINVAL;
-+ }
-+ printk(KERN_INFO "md/raid:%s: reshape will continue\n",
-+ mdname(mddev));
-+ /* OK, we should be able to continue; */
-+ } else {
-+ BUG_ON(mddev->level != mddev->new_level);
-+ BUG_ON(mddev->layout != mddev->new_layout);
-+ BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
-+ BUG_ON(mddev->delta_disks != 0);
-+ }
-+
-+ if (mddev->private == NULL)
-+ conf = setup_conf(mddev);
-+ else
-+ conf = mddev->private;
-+
-+ if (IS_ERR(conf))
-+ return PTR_ERR(conf);
-+
-+ conf->min_offset_diff = min_offset_diff;
-+ mddev->thread = conf->thread;
-+ conf->thread = NULL;
-+ mddev->private = conf;
-+
-+ for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
-+ i++) {
-+ rdev = conf->disks[i].rdev;
-+ if (!rdev && conf->disks[i].replacement) {
-+ /* The replacement is all we have yet */
-+ rdev = conf->disks[i].replacement;
-+ conf->disks[i].replacement = NULL;
-+ clear_bit(Replacement, &rdev->flags);
-+ conf->disks[i].rdev = rdev;
-+ }
-+ if (!rdev)
-+ continue;
-+ if (conf->disks[i].replacement &&
-+ conf->reshape_progress != MaxSector) {
-+ /* replacements and reshape simply do not mix. */
-+ printk(KERN_ERR "md: cannot handle concurrent "
-+ "replacement and reshape.\n");
-+ goto abort;
-+ }
-+ if (test_bit(In_sync, &rdev->flags)) {
-+ working_disks++;
-+ continue;
-+ }
-+ /* This disc is not fully in-sync. However if it
-+ * just stored parity (beyond the recovery_offset),
-+ * when we don't need to be concerned about the
-+ * array being dirty.
-+ * When reshape goes 'backwards', we never have
-+ * partially completed devices, so we only need
-+ * to worry about reshape going forwards.
-+ */
-+ /* Hack because v0.91 doesn't store recovery_offset properly. */
-+ if (mddev->major_version == 0 &&
-+ mddev->minor_version > 90)
-+ rdev->recovery_offset = reshape_offset;
-+
-+ if (rdev->recovery_offset < reshape_offset) {
-+ /* We need to check old and new layout */
-+ if (!only_parity(rdev->raid_disk,
-+ conf->algorithm,
-+ conf->raid_disks,
-+ conf->max_degraded))
-+ continue;
-+ }
-+ if (!only_parity(rdev->raid_disk,
-+ conf->prev_algo,
-+ conf->previous_raid_disks,
-+ conf->max_degraded))
-+ continue;
-+ dirty_parity_disks++;
-+ }
-+
-+ /*
-+ * 0 for a fully functional array, 1 or 2 for a degraded array.
-+ */
-+ mddev->degraded = calc_degraded(conf);
-+
-+ if (has_failed(conf)) {
-+ printk(KERN_ERR "md/raid:%s: not enough operational devices"
-+ " (%d/%d failed)\n",
-+ mdname(mddev), mddev->degraded, conf->raid_disks);
-+ goto abort;
-+ }
-+
-+ /* device size must be a multiple of chunk size */
-+ mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
-+ mddev->resync_max_sectors = mddev->dev_sectors;
-+
-+ if (mddev->degraded > dirty_parity_disks &&
-+ mddev->recovery_cp != MaxSector) {
-+ if (mddev->ok_start_degraded)
-+ printk(KERN_WARNING
-+ "md/raid:%s: starting dirty degraded array"
-+ " - data corruption possible.\n",
-+ mdname(mddev));
-+ else {
-+ printk(KERN_ERR
-+ "md/raid:%s: cannot start dirty degraded array.\n",
-+ mdname(mddev));
-+ goto abort;
-+ }
-+ }
-+
-+ if (mddev->degraded == 0)
-+ printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
-+ " devices, algorithm %d\n", mdname(mddev), conf->level,
-+ mddev->raid_disks-mddev->degraded, mddev->raid_disks,
-+ mddev->new_layout);
-+ else
-+ printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
-+ " out of %d devices, algorithm %d\n",
-+ mdname(mddev), conf->level,
-+ mddev->raid_disks - mddev->degraded,
-+ mddev->raid_disks, mddev->new_layout);
-+
-+ print_raid5_conf(conf);
-+
-+ if (conf->reshape_progress != MaxSector) {
-+ conf->reshape_safe = conf->reshape_progress;
-+ atomic_set(&conf->reshape_stripes, 0);
-+ clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
-+ clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
-+ set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
-+ set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
-+ mddev->sync_thread = md_register_thread(md_do_sync, mddev,
-+ "reshape");
-+ }
-+
-+ /* Ok, everything is just fine now */
-+ if (mddev->to_remove == &raid5_attrs_group)
-+ mddev->to_remove = NULL;
-+ else if (mddev->kobj.sd &&
-+ sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
-+ printk(KERN_WARNING
-+ "raid5: failed to create sysfs attributes for %s\n",
-+ mdname(mddev));
-+ md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
-+
-+ if (mddev->queue) {
-+ int chunk_size;
-+ bool discard_supported = true;
-+ /* read-ahead size must cover two whole stripes, which
-+ * is 2 * (datadisks) * chunksize where 'n' is the
-+ * number of raid devices
-+ */
-+ int data_disks = conf->previous_raid_disks - conf->max_degraded;
-+ int stripe = data_disks *
-+ ((mddev->chunk_sectors << 9) / PAGE_SIZE);
-+ if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
-+ mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
-+
-+ chunk_size = mddev->chunk_sectors << 9;
-+ blk_queue_io_min(mddev->queue, chunk_size);
-+ blk_queue_io_opt(mddev->queue, chunk_size *
-+ (conf->raid_disks - conf->max_degraded));
-+ mddev->queue->limits.raid_partial_stripes_expensive = 1;
-+ /*
-+ * We can only discard a whole stripe. It doesn't make sense to
-+ * discard data disk but write parity disk
-+ */
-+ stripe = stripe * PAGE_SIZE;
-+ /* Round up to power of 2, as discard handling
-+ * currently assumes that */
-+ while ((stripe-1) & stripe)
-+ stripe = (stripe | (stripe-1)) + 1;
-+ mddev->queue->limits.discard_alignment = stripe;
-+ mddev->queue->limits.discard_granularity = stripe;
-+ /*
-+ * unaligned part of discard request will be ignored, so can't
-+ * guarantee discard_zeroes_data
-+ */
-+ mddev->queue->limits.discard_zeroes_data = 0;
-+
-+ blk_queue_max_write_same_sectors(mddev->queue, 0);
-+
-+ rdev_for_each(rdev, mddev) {
-+ disk_stack_limits(mddev->gendisk, rdev->bdev,
-+ rdev->data_offset << 9);
-+ disk_stack_limits(mddev->gendisk, rdev->bdev,
-+ rdev->new_data_offset << 9);
-+ /*
-+ * discard_zeroes_data is required, otherwise data
-+ * could be lost. Consider a scenario: discard a stripe
-+ * (the stripe could be inconsistent if
-+ * discard_zeroes_data is 0); write one disk of the
-+ * stripe (the stripe could be inconsistent again
-+ * depending on which disks are used to calculate
-+ * parity); the disk is broken; The stripe data of this
-+ * disk is lost.
-+ */
-+ if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) ||
-+ !bdev_get_queue(rdev->bdev)->
-+ limits.discard_zeroes_data)
-+ discard_supported = false;
-+ /* Unfortunately, discard_zeroes_data is not currently
-+ * a guarantee - just a hint. So we only allow DISCARD
-+ * if the sysadmin has confirmed that only safe devices
-+ * are in use by setting a module parameter.
-+ */
-+ if (!devices_handle_discard_safely) {
-+ if (discard_supported) {
-+ pr_info("md/raid456: discard support disabled due to uncertainty.\n");
-+ pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n");
-+ }
-+ discard_supported = false;
-+ }
-+ }
-+
-+ if (discard_supported &&
-+ mddev->queue->limits.max_discard_sectors >= stripe &&
-+ mddev->queue->limits.discard_granularity >= stripe)
-+ queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
-+ mddev->queue);
-+ else
-+ queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
-+ mddev->queue);
-+ }
-+
-+ return 0;
-+abort:
-+ md_unregister_thread(&mddev->thread);
-+ print_raid5_conf(conf);
-+ free_conf(conf);
-+ mddev->private = NULL;
-+ printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
-+ return -EIO;
-+}
-+
-+static void raid5_free(struct mddev *mddev, void *priv)
-+{
-+ struct r5conf *conf = priv;
-+
-+ free_conf(conf);
-+ mddev->to_remove = &raid5_attrs_group;
-+}
-+
-+static void status(struct seq_file *seq, struct mddev *mddev)
-+{
-+ struct r5conf *conf = mddev->private;
-+ int i;
-+
-+ seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
-+ mddev->chunk_sectors / 2, mddev->layout);
-+ seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
-+ for (i = 0; i < conf->raid_disks; i++)
-+ seq_printf (seq, "%s",
-+ conf->disks[i].rdev &&
-+ test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
-+ seq_printf (seq, "]");
-+}
-+
-+static void print_raid5_conf (struct r5conf *conf)
-+{
-+ int i;
-+ struct disk_info *tmp;
-+
-+ printk(KERN_DEBUG "RAID conf printout:\n");
-+ if (!conf) {
-+ printk("(conf==NULL)\n");
-+ return;
-+ }
-+ printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
-+ conf->raid_disks,
-+ conf->raid_disks - conf->mddev->degraded);
-+
-+ for (i = 0; i < conf->raid_disks; i++) {
-+ char b[BDEVNAME_SIZE];
-+ tmp = conf->disks + i;
-+ if (tmp->rdev)
-+ printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
-+ i, !test_bit(Faulty, &tmp->rdev->flags),
-+ bdevname(tmp->rdev->bdev, b));
-+ }
-+}
-+
-+static int raid5_spare_active(struct mddev *mddev)
-+{
-+ int i;
-+ struct r5conf *conf = mddev->private;
-+ struct disk_info *tmp;
-+ int count = 0;
-+ unsigned long flags;
-+
-+ for (i = 0; i < conf->raid_disks; i++) {
-+ tmp = conf->disks + i;
-+ if (tmp->replacement
-+ && tmp->replacement->recovery_offset == MaxSector
-+ && !test_bit(Faulty, &tmp->replacement->flags)
-+ && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
-+ /* Replacement has just become active. */
-+ if (!tmp->rdev
-+ || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
-+ count++;
-+ if (tmp->rdev) {
-+ /* Replaced device not technically faulty,
-+ * but we need to be sure it gets removed
-+ * and never re-added.
-+ */
-+ set_bit(Faulty, &tmp->rdev->flags);
-+ sysfs_notify_dirent_safe(
-+ tmp->rdev->sysfs_state);
-+ }
-+ sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
-+ } else if (tmp->rdev
-+ && tmp->rdev->recovery_offset == MaxSector
-+ && !test_bit(Faulty, &tmp->rdev->flags)
-+ && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
-+ count++;
-+ sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
-+ }
-+ }
-+ spin_lock_irqsave(&conf->device_lock, flags);
-+ mddev->degraded = calc_degraded(conf);
-+ spin_unlock_irqrestore(&conf->device_lock, flags);
-+ print_raid5_conf(conf);
-+ return count;
-+}
-+
-+static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
-+{
-+ struct r5conf *conf = mddev->private;
-+ int err = 0;
-+ int number = rdev->raid_disk;
-+ struct md_rdev **rdevp;
-+ struct disk_info *p = conf->disks + number;
-+
-+ print_raid5_conf(conf);
-+ if (rdev == p->rdev)
-+ rdevp = &p->rdev;
-+ else if (rdev == p->replacement)
-+ rdevp = &p->replacement;
-+ else
-+ return 0;
-+
-+ if (number >= conf->raid_disks &&
-+ conf->reshape_progress == MaxSector)
-+ clear_bit(In_sync, &rdev->flags);
-+
-+ if (test_bit(In_sync, &rdev->flags) ||
-+ atomic_read(&rdev->nr_pending)) {
-+ err = -EBUSY;
-+ goto abort;
-+ }
-+ /* Only remove non-faulty devices if recovery
-+ * isn't possible.
-+ */
-+ if (!test_bit(Faulty, &rdev->flags) &&
-+ mddev->recovery_disabled != conf->recovery_disabled &&
-+ !has_failed(conf) &&
-+ (!p->replacement || p->replacement == rdev) &&
-+ number < conf->raid_disks) {
-+ err = -EBUSY;
-+ goto abort;
-+ }
-+ *rdevp = NULL;
-+ synchronize_rcu();
-+ if (atomic_read(&rdev->nr_pending)) {
-+ /* lost the race, try later */
-+ err = -EBUSY;
-+ *rdevp = rdev;
-+ } else if (p->replacement) {
-+ /* We must have just cleared 'rdev' */
-+ p->rdev = p->replacement;
-+ clear_bit(Replacement, &p->replacement->flags);
-+ smp_mb(); /* Make sure other CPUs may see both as identical
-+ * but will never see neither - if they are careful
-+ */
-+ p->replacement = NULL;
-+ clear_bit(WantReplacement, &rdev->flags);
-+ } else
-+ /* We might have just removed the Replacement as faulty-
-+ * clear the bit just in case
-+ */
-+ clear_bit(WantReplacement, &rdev->flags);
-+abort:
-+
-+ print_raid5_conf(conf);
-+ return err;
-+}
-+
-+static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
-+{
-+ struct r5conf *conf = mddev->private;
-+ int err = -EEXIST;
-+ int disk;
-+ struct disk_info *p;
-+ int first = 0;
-+ int last = conf->raid_disks - 1;
-+
-+ if (mddev->recovery_disabled == conf->recovery_disabled)
-+ return -EBUSY;
-+
-+ if (rdev->saved_raid_disk < 0 && has_failed(conf))
-+ /* no point adding a device */
-+ return -EINVAL;
-+
-+ if (rdev->raid_disk >= 0)
-+ first = last = rdev->raid_disk;
-+
-+ /*
-+ * find the disk ... but prefer rdev->saved_raid_disk
-+ * if possible.
-+ */
-+ if (rdev->saved_raid_disk >= 0 &&
-+ rdev->saved_raid_disk >= first &&
-+ conf->disks[rdev->saved_raid_disk].rdev == NULL)
-+ first = rdev->saved_raid_disk;
-+
-+ for (disk = first; disk <= last; disk++) {
-+ p = conf->disks + disk;
-+ if (p->rdev == NULL) {
-+ clear_bit(In_sync, &rdev->flags);
-+ rdev->raid_disk = disk;
-+ err = 0;
-+ if (rdev->saved_raid_disk != disk)
-+ conf->fullsync = 1;
-+ rcu_assign_pointer(p->rdev, rdev);
-+ goto out;
-+ }
-+ }
-+ for (disk = first; disk <= last; disk++) {
-+ p = conf->disks + disk;
-+ if (test_bit(WantReplacement, &p->rdev->flags) &&
-+ p->replacement == NULL) {
-+ clear_bit(In_sync, &rdev->flags);
-+ set_bit(Replacement, &rdev->flags);
-+ rdev->raid_disk = disk;
-+ err = 0;
-+ conf->fullsync = 1;
-+ rcu_assign_pointer(p->replacement, rdev);
-+ break;
-+ }
-+ }
-+out:
-+ print_raid5_conf(conf);
-+ return err;
-+}
-+
-+static int raid5_resize(struct mddev *mddev, sector_t sectors)
-+{
-+ /* no resync is happening, and there is enough space
-+ * on all devices, so we can resize.
-+ * We need to make sure resync covers any new space.
-+ * If the array is shrinking we should possibly wait until
-+ * any io in the removed space completes, but it hardly seems
-+ * worth it.
-+ */
-+ sector_t newsize;
-+ sectors &= ~((sector_t)mddev->chunk_sectors - 1);
-+ newsize = raid5_size(mddev, sectors, mddev->raid_disks);
-+ if (mddev->external_size &&
-+ mddev->array_sectors > newsize)
-+ return -EINVAL;
-+ if (mddev->bitmap) {
-+ int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0);
-+ if (ret)
-+ return ret;
-+ }
-+ md_set_array_sectors(mddev, newsize);
-+ set_capacity(mddev->gendisk, mddev->array_sectors);
-+ revalidate_disk(mddev->gendisk);
-+ if (sectors > mddev->dev_sectors &&
-+ mddev->recovery_cp > mddev->dev_sectors) {
-+ mddev->recovery_cp = mddev->dev_sectors;
-+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
-+ }
-+ mddev->dev_sectors = sectors;
-+ mddev->resync_max_sectors = sectors;
-+ return 0;
-+}
-+
-+static int check_stripe_cache(struct mddev *mddev)
-+{
-+ /* Can only proceed if there are plenty of stripe_heads.
-+ * We need a minimum of one full stripe,, and for sensible progress
-+ * it is best to have about 4 times that.
-+ * If we require 4 times, then the default 256 4K stripe_heads will
-+ * allow for chunk sizes up to 256K, which is probably OK.
-+ * If the chunk size is greater, user-space should request more
-+ * stripe_heads first.
-+ */
-+ struct r5conf *conf = mddev->private;
-+ if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
-+ > conf->min_nr_stripes ||
-+ ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
-+ > conf->min_nr_stripes) {
-+ printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n",
-+ mdname(mddev),
-+ ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
-+ / STRIPE_SIZE)*4);
-+ return 0;
-+ }
-+ return 1;
-+}
-+
-+static int check_reshape(struct mddev *mddev)
-+{
-+ struct r5conf *conf = mddev->private;
-+
-+ if (mddev->delta_disks == 0 &&
-+ mddev->new_layout == mddev->layout &&
-+ mddev->new_chunk_sectors == mddev->chunk_sectors)
-+ return 0; /* nothing to do */
-+ if (has_failed(conf))
-+ return -EINVAL;
-+ if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
-+ /* We might be able to shrink, but the devices must
-+ * be made bigger first.
-+ * For raid6, 4 is the minimum size.
-+ * Otherwise 2 is the minimum
-+ */
-+ int min = 2;
-+ if (mddev->level == 6)
-+ min = 4;
-+ if (mddev->raid_disks + mddev->delta_disks < min)
-+ return -EINVAL;
-+ }
-+
-+ if (!check_stripe_cache(mddev))
-+ return -ENOSPC;
-+
-+ if (mddev->new_chunk_sectors > mddev->chunk_sectors ||
-+ mddev->delta_disks > 0)
-+ if (resize_chunks(conf,
-+ conf->previous_raid_disks
-+ + max(0, mddev->delta_disks),
-+ max(mddev->new_chunk_sectors,
-+ mddev->chunk_sectors)
-+ ) < 0)
-+ return -ENOMEM;
-+ return resize_stripes(conf, (conf->previous_raid_disks
-+ + mddev->delta_disks));
-+}
-+
-+static int raid5_start_reshape(struct mddev *mddev)
-+{
-+ struct r5conf *conf = mddev->private;
-+ struct md_rdev *rdev;
-+ int spares = 0;
-+ unsigned long flags;
-+
-+ if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
-+ return -EBUSY;
-+
-+ if (!check_stripe_cache(mddev))
-+ return -ENOSPC;
-+
-+ if (has_failed(conf))
-+ return -EINVAL;
-+
-+ rdev_for_each(rdev, mddev) {
-+ if (!test_bit(In_sync, &rdev->flags)
-+ && !test_bit(Faulty, &rdev->flags))
-+ spares++;
-+ }
-+
-+ if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
-+ /* Not enough devices even to make a degraded array
-+ * of that size
-+ */
-+ return -EINVAL;
-+
-+ /* Refuse to reduce size of the array. Any reductions in
-+ * array size must be through explicit setting of array_size
-+ * attribute.
-+ */
-+ if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
-+ < mddev->array_sectors) {
-+ printk(KERN_ERR "md/raid:%s: array size must be reduced "
-+ "before number of disks\n", mdname(mddev));
-+ return -EINVAL;
-+ }
-+
-+ atomic_set(&conf->reshape_stripes, 0);
-+ spin_lock_irq(&conf->device_lock);
-+ write_seqcount_begin(&conf->gen_lock);
-+ conf->previous_raid_disks = conf->raid_disks;
-+ conf->raid_disks += mddev->delta_disks;
-+ conf->prev_chunk_sectors = conf->chunk_sectors;
-+ conf->chunk_sectors = mddev->new_chunk_sectors;
-+ conf->prev_algo = conf->algorithm;
-+ conf->algorithm = mddev->new_layout;
-+ conf->generation++;
-+ /* Code that selects data_offset needs to see the generation update
-+ * if reshape_progress has been set - so a memory barrier needed.
-+ */
-+ smp_mb();
-+ if (mddev->reshape_backwards)
-+ conf->reshape_progress = raid5_size(mddev, 0, 0);
-+ else
-+ conf->reshape_progress = 0;
-+ conf->reshape_safe = conf->reshape_progress;
-+ write_seqcount_end(&conf->gen_lock);
-+ spin_unlock_irq(&conf->device_lock);
-+
-+ /* Now make sure any requests that proceeded on the assumption
-+ * the reshape wasn't running - like Discard or Read - have
-+ * completed.
-+ */
-+ mddev_suspend(mddev);
-+ mddev_resume(mddev);
-+
-+ /* Add some new drives, as many as will fit.
-+ * We know there are enough to make the newly sized array work.
-+ * Don't add devices if we are reducing the number of
-+ * devices in the array. This is because it is not possible
-+ * to correctly record the "partially reconstructed" state of
-+ * such devices during the reshape and confusion could result.
-+ */
-+ if (mddev->delta_disks >= 0) {
-+ rdev_for_each(rdev, mddev)
-+ if (rdev->raid_disk < 0 &&
-+ !test_bit(Faulty, &rdev->flags)) {
-+ if (raid5_add_disk(mddev, rdev) == 0) {
-+ if (rdev->raid_disk
-+ >= conf->previous_raid_disks)
-+ set_bit(In_sync, &rdev->flags);
-+ else
-+ rdev->recovery_offset = 0;
-+
-+ if (sysfs_link_rdev(mddev, rdev))
-+ /* Failure here is OK */;
-+ }
-+ } else if (rdev->raid_disk >= conf->previous_raid_disks
-+ && !test_bit(Faulty, &rdev->flags)) {
-+ /* This is a spare that was manually added */
-+ set_bit(In_sync, &rdev->flags);
-+ }
-+
-+ /* When a reshape changes the number of devices,
-+ * ->degraded is measured against the larger of the
-+ * pre and post number of devices.
-+ */
-+ spin_lock_irqsave(&conf->device_lock, flags);
-+ mddev->degraded = calc_degraded(conf);
-+ spin_unlock_irqrestore(&conf->device_lock, flags);
-+ }
-+ mddev->raid_disks = conf->raid_disks;
-+ mddev->reshape_position = conf->reshape_progress;
-+ set_bit(MD_CHANGE_DEVS, &mddev->flags);
-+
-+ clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
-+ clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
-+ clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
-+ set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
-+ set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
-+ mddev->sync_thread = md_register_thread(md_do_sync, mddev,
-+ "reshape");
-+ if (!mddev->sync_thread) {
-+ mddev->recovery = 0;
-+ spin_lock_irq(&conf->device_lock);
-+ write_seqcount_begin(&conf->gen_lock);
-+ mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
-+ mddev->new_chunk_sectors =
-+ conf->chunk_sectors = conf->prev_chunk_sectors;
-+ mddev->new_layout = conf->algorithm = conf->prev_algo;
-+ rdev_for_each(rdev, mddev)
-+ rdev->new_data_offset = rdev->data_offset;
-+ smp_wmb();
-+ conf->generation --;
-+ conf->reshape_progress = MaxSector;
-+ mddev->reshape_position = MaxSector;
-+ write_seqcount_end(&conf->gen_lock);
-+ spin_unlock_irq(&conf->device_lock);
-+ return -EAGAIN;
-+ }
-+ conf->reshape_checkpoint = jiffies;
-+ md_wakeup_thread(mddev->sync_thread);
-+ md_new_event(mddev);
-+ return 0;
-+}
-+
-+/* This is called from the reshape thread and should make any
-+ * changes needed in 'conf'
-+ */
-+static void end_reshape(struct r5conf *conf)
-+{
-+
-+ if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
-+ struct md_rdev *rdev;
-+
-+ spin_lock_irq(&conf->device_lock);
-+ conf->previous_raid_disks = conf->raid_disks;
-+ rdev_for_each(rdev, conf->mddev)
-+ rdev->data_offset = rdev->new_data_offset;
-+ smp_wmb();
-+ conf->reshape_progress = MaxSector;
-+ spin_unlock_irq(&conf->device_lock);
-+ wake_up(&conf->wait_for_overlap);
-+
-+ /* read-ahead size must cover two whole stripes, which is
-+ * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
-+ */
-+ if (conf->mddev->queue) {
-+ int data_disks = conf->raid_disks - conf->max_degraded;
-+ int stripe = data_disks * ((conf->chunk_sectors << 9)
-+ / PAGE_SIZE);
-+ if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
-+ conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
-+ }
-+ }
-+}
-+
-+/* This is called from the raid5d thread with mddev_lock held.
-+ * It makes config changes to the device.
-+ */
-+static void raid5_finish_reshape(struct mddev *mddev)
-+{
-+ struct r5conf *conf = mddev->private;
-+
-+ if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
-+
-+ if (mddev->delta_disks > 0) {
-+ md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
-+ set_capacity(mddev->gendisk, mddev->array_sectors);
-+ revalidate_disk(mddev->gendisk);
-+ } else {
-+ int d;
-+ spin_lock_irq(&conf->device_lock);
-+ mddev->degraded = calc_degraded(conf);
-+ spin_unlock_irq(&conf->device_lock);
-+ for (d = conf->raid_disks ;
-+ d < conf->raid_disks - mddev->delta_disks;
-+ d++) {
-+ struct md_rdev *rdev = conf->disks[d].rdev;
-+ if (rdev)
-+ clear_bit(In_sync, &rdev->flags);
-+ rdev = conf->disks[d].replacement;
-+ if (rdev)
-+ clear_bit(In_sync, &rdev->flags);
-+ }
-+ }
-+ mddev->layout = conf->algorithm;
-+ mddev->chunk_sectors = conf->chunk_sectors;
-+ mddev->reshape_position = MaxSector;
-+ mddev->delta_disks = 0;
-+ mddev->reshape_backwards = 0;
-+ }
-+}
-+
-+static void raid5_quiesce(struct mddev *mddev, int state)
-+{
-+ struct r5conf *conf = mddev->private;
-+
-+ switch(state) {
-+ case 2: /* resume for a suspend */
-+ wake_up(&conf->wait_for_overlap);
-+ break;
-+
-+ case 1: /* stop all writes */
-+ lock_all_device_hash_locks_irq(conf);
-+ /* '2' tells resync/reshape to pause so that all
-+ * active stripes can drain
-+ */
-+ conf->quiesce = 2;
-+ wait_event_cmd(conf->wait_for_stripe,
-+ atomic_read(&conf->active_stripes) == 0 &&
-+ atomic_read(&conf->active_aligned_reads) == 0,
-+ unlock_all_device_hash_locks_irq(conf),
-+ lock_all_device_hash_locks_irq(conf));
-+ conf->quiesce = 1;
-+ unlock_all_device_hash_locks_irq(conf);
-+ /* allow reshape to continue */
-+ wake_up(&conf->wait_for_overlap);
-+ break;
-+
-+ case 0: /* re-enable writes */
-+ lock_all_device_hash_locks_irq(conf);
-+ conf->quiesce = 0;
-+ wake_up(&conf->wait_for_stripe);
-+ wake_up(&conf->wait_for_overlap);
-+ unlock_all_device_hash_locks_irq(conf);
-+ break;
-+ }
-+}
-+
-+static void *raid45_takeover_raid0(struct mddev *mddev, int level)
-+{
-+ struct r0conf *raid0_conf = mddev->private;
-+ sector_t sectors;
-+
-+ /* for raid0 takeover only one zone is supported */
-+ if (raid0_conf->nr_strip_zones > 1) {
-+ printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
-+ mdname(mddev));
-+ return ERR_PTR(-EINVAL);
-+ }
-+
-+ sectors = raid0_conf->strip_zone[0].zone_end;
-+ sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
-+ mddev->dev_sectors = sectors;
-+ mddev->new_level = level;
-+ mddev->new_layout = ALGORITHM_PARITY_N;
-+ mddev->new_chunk_sectors = mddev->chunk_sectors;
-+ mddev->raid_disks += 1;
-+ mddev->delta_disks = 1;
-+ /* make sure it will be not marked as dirty */
-+ mddev->recovery_cp = MaxSector;
-+
-+ return setup_conf(mddev);
-+}
-+
-+static void *raid5_takeover_raid1(struct mddev *mddev)
-+{
-+ int chunksect;
-+
-+ if (mddev->raid_disks != 2 ||
-+ mddev->degraded > 1)
-+ return ERR_PTR(-EINVAL);
-+
-+ /* Should check if there are write-behind devices? */
-+
-+ chunksect = 64*2; /* 64K by default */
-+
-+ /* The array must be an exact multiple of chunksize */
-+ while (chunksect && (mddev->array_sectors & (chunksect-1)))
-+ chunksect >>= 1;
-+
-+ if ((chunksect<<9) < STRIPE_SIZE)
-+ /* array size does not allow a suitable chunk size */
-+ return ERR_PTR(-EINVAL);
-+
-+ mddev->new_level = 5;
-+ mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
-+ mddev->new_chunk_sectors = chunksect;
-+
-+ return setup_conf(mddev);
-+}
-+
-+static void *raid5_takeover_raid6(struct mddev *mddev)
-+{
-+ int new_layout;
-+
-+ switch (mddev->layout) {
-+ case ALGORITHM_LEFT_ASYMMETRIC_6:
-+ new_layout = ALGORITHM_LEFT_ASYMMETRIC;
-+ break;
-+ case ALGORITHM_RIGHT_ASYMMETRIC_6:
-+ new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
-+ break;
-+ case ALGORITHM_LEFT_SYMMETRIC_6:
-+ new_layout = ALGORITHM_LEFT_SYMMETRIC;
-+ break;
-+ case ALGORITHM_RIGHT_SYMMETRIC_6:
-+ new_layout = ALGORITHM_RIGHT_SYMMETRIC;
-+ break;
-+ case ALGORITHM_PARITY_0_6:
-+ new_layout = ALGORITHM_PARITY_0;
-+ break;
-+ case ALGORITHM_PARITY_N:
-+ new_layout = ALGORITHM_PARITY_N;
-+ break;
-+ default:
-+ return ERR_PTR(-EINVAL);
-+ }
-+ mddev->new_level = 5;
-+ mddev->new_layout = new_layout;
-+ mddev->delta_disks = -1;
-+ mddev->raid_disks -= 1;
-+ return setup_conf(mddev);
-+}
-+
-+static int raid5_check_reshape(struct mddev *mddev)
-+{
-+ /* For a 2-drive array, the layout and chunk size can be changed
-+ * immediately as not restriping is needed.
-+ * For larger arrays we record the new value - after validation
-+ * to be used by a reshape pass.
-+ */
-+ struct r5conf *conf = mddev->private;
-+ int new_chunk = mddev->new_chunk_sectors;
-+
-+ if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
-+ return -EINVAL;
-+ if (new_chunk > 0) {
-+ if (!is_power_of_2(new_chunk))
-+ return -EINVAL;
-+ if (new_chunk < (PAGE_SIZE>>9))
-+ return -EINVAL;
-+ if (mddev->array_sectors & (new_chunk-1))
-+ /* not factor of array size */
-+ return -EINVAL;
-+ }
-+
-+ /* They look valid */
-+
-+ if (mddev->raid_disks == 2) {
-+ /* can make the change immediately */
-+ if (mddev->new_layout >= 0) {
-+ conf->algorithm = mddev->new_layout;
-+ mddev->layout = mddev->new_layout;
-+ }
-+ if (new_chunk > 0) {
-+ conf->chunk_sectors = new_chunk ;
-+ mddev->chunk_sectors = new_chunk;
-+ }
-+ set_bit(MD_CHANGE_DEVS, &mddev->flags);
-+ md_wakeup_thread(mddev->thread);
-+ }
-+ return check_reshape(mddev);
-+}
-+
-+static int raid6_check_reshape(struct mddev *mddev)
-+{
-+ int new_chunk = mddev->new_chunk_sectors;
-+
-+ if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
-+ return -EINVAL;
-+ if (new_chunk > 0) {
-+ if (!is_power_of_2(new_chunk))
-+ return -EINVAL;
-+ if (new_chunk < (PAGE_SIZE >> 9))
-+ return -EINVAL;
-+ if (mddev->array_sectors & (new_chunk-1))
-+ /* not factor of array size */
-+ return -EINVAL;
-+ }
-+
-+ /* They look valid */
-+ return check_reshape(mddev);
-+}
-+
-+static void *raid5_takeover(struct mddev *mddev)
-+{
-+ /* raid5 can take over:
-+ * raid0 - if there is only one strip zone - make it a raid4 layout
-+ * raid1 - if there are two drives. We need to know the chunk size
-+ * raid4 - trivial - just use a raid4 layout.
-+ * raid6 - Providing it is a *_6 layout
-+ */
-+ if (mddev->level == 0)
-+ return raid45_takeover_raid0(mddev, 5);
-+ if (mddev->level == 1)
-+ return raid5_takeover_raid1(mddev);
-+ if (mddev->level == 4) {
-+ mddev->new_layout = ALGORITHM_PARITY_N;
-+ mddev->new_level = 5;
-+ return setup_conf(mddev);
-+ }
-+ if (mddev->level == 6)
-+ return raid5_takeover_raid6(mddev);
-+
-+ return ERR_PTR(-EINVAL);
-+}
-+
-+static void *raid4_takeover(struct mddev *mddev)
-+{
-+ /* raid4 can take over:
-+ * raid0 - if there is only one strip zone
-+ * raid5 - if layout is right
-+ */
-+ if (mddev->level == 0)
-+ return raid45_takeover_raid0(mddev, 4);
-+ if (mddev->level == 5 &&
-+ mddev->layout == ALGORITHM_PARITY_N) {
-+ mddev->new_layout = 0;
-+ mddev->new_level = 4;
-+ return setup_conf(mddev);
-+ }
-+ return ERR_PTR(-EINVAL);
-+}
-+
-+static struct md_personality raid5_personality;
-+
-+static void *raid6_takeover(struct mddev *mddev)
-+{
-+ /* Currently can only take over a raid5. We map the
-+ * personality to an equivalent raid6 personality
-+ * with the Q block at the end.
-+ */
-+ int new_layout;
-+
-+ if (mddev->pers != &raid5_personality)
-+ return ERR_PTR(-EINVAL);
-+ if (mddev->degraded > 1)
-+ return ERR_PTR(-EINVAL);
-+ if (mddev->raid_disks > 253)
-+ return ERR_PTR(-EINVAL);
-+ if (mddev->raid_disks < 3)
-+ return ERR_PTR(-EINVAL);
-+
-+ switch (mddev->layout) {
-+ case ALGORITHM_LEFT_ASYMMETRIC:
-+ new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
-+ break;
-+ case ALGORITHM_RIGHT_ASYMMETRIC:
-+ new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
-+ break;
-+ case ALGORITHM_LEFT_SYMMETRIC:
-+ new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
-+ break;
-+ case ALGORITHM_RIGHT_SYMMETRIC:
-+ new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
-+ break;
-+ case ALGORITHM_PARITY_0:
-+ new_layout = ALGORITHM_PARITY_0_6;
-+ break;
-+ case ALGORITHM_PARITY_N:
-+ new_layout = ALGORITHM_PARITY_N;
-+ break;
-+ default:
-+ return ERR_PTR(-EINVAL);
-+ }
-+ mddev->new_level = 6;
-+ mddev->new_layout = new_layout;
-+ mddev->delta_disks = 1;
-+ mddev->raid_disks += 1;
-+ return setup_conf(mddev);
-+}
-+
-+static struct md_personality raid6_personality =
-+{
-+ .name = "raid6",
-+ .level = 6,
-+ .owner = THIS_MODULE,
-+ .make_request = make_request,
-+ .run = run,
-+ .free = raid5_free,
-+ .status = status,
-+ .error_handler = error,
-+ .hot_add_disk = raid5_add_disk,
-+ .hot_remove_disk= raid5_remove_disk,
-+ .spare_active = raid5_spare_active,
-+ .sync_request = sync_request,
-+ .resize = raid5_resize,
-+ .size = raid5_size,
-+ .check_reshape = raid6_check_reshape,
-+ .start_reshape = raid5_start_reshape,
-+ .finish_reshape = raid5_finish_reshape,
-+ .quiesce = raid5_quiesce,
-+ .takeover = raid6_takeover,
-+ .congested = raid5_congested,
-+ .mergeable_bvec = raid5_mergeable_bvec,
-+};
-+static struct md_personality raid5_personality =
-+{
-+ .name = "raid5",
-+ .level = 5,
-+ .owner = THIS_MODULE,
-+ .make_request = make_request,
-+ .run = run,
-+ .free = raid5_free,
-+ .status = status,
-+ .error_handler = error,
-+ .hot_add_disk = raid5_add_disk,
-+ .hot_remove_disk= raid5_remove_disk,
-+ .spare_active = raid5_spare_active,
-+ .sync_request = sync_request,
-+ .resize = raid5_resize,
-+ .size = raid5_size,
-+ .check_reshape = raid5_check_reshape,
-+ .start_reshape = raid5_start_reshape,
-+ .finish_reshape = raid5_finish_reshape,
-+ .quiesce = raid5_quiesce,
-+ .takeover = raid5_takeover,
-+ .congested = raid5_congested,
-+ .mergeable_bvec = raid5_mergeable_bvec,
-+};
-+
-+static struct md_personality raid4_personality =
-+{
-+ .name = "raid4",
-+ .level = 4,
-+ .owner = THIS_MODULE,
-+ .make_request = make_request,
-+ .run = run,
-+ .free = raid5_free,
-+ .status = status,
-+ .error_handler = error,
-+ .hot_add_disk = raid5_add_disk,
-+ .hot_remove_disk= raid5_remove_disk,
-+ .spare_active = raid5_spare_active,
-+ .sync_request = sync_request,
-+ .resize = raid5_resize,
-+ .size = raid5_size,
-+ .check_reshape = raid5_check_reshape,
-+ .start_reshape = raid5_start_reshape,
-+ .finish_reshape = raid5_finish_reshape,
-+ .quiesce = raid5_quiesce,
-+ .takeover = raid4_takeover,
-+ .congested = raid5_congested,
-+ .mergeable_bvec = raid5_mergeable_bvec,
-+};
-+
-+static int __init raid5_init(void)
-+{
-+ raid5_wq = alloc_workqueue("raid5wq",
-+ WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
-+ if (!raid5_wq)
-+ return -ENOMEM;
-+ register_md_personality(&raid6_personality);
-+ register_md_personality(&raid5_personality);
-+ register_md_personality(&raid4_personality);
-+ return 0;
-+}
-+
-+static void raid5_exit(void)
-+{
-+ unregister_md_personality(&raid6_personality);
-+ unregister_md_personality(&raid5_personality);
-+ unregister_md_personality(&raid4_personality);
-+ destroy_workqueue(raid5_wq);
-+}
-+
-+module_init(raid5_init);
-+module_exit(raid5_exit);
-+MODULE_LICENSE("GPL");
-+MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
-+MODULE_ALIAS("md-personality-4"); /* RAID5 */
-+MODULE_ALIAS("md-raid5");
-+MODULE_ALIAS("md-raid4");
-+MODULE_ALIAS("md-level-5");
-+MODULE_ALIAS("md-level-4");
-+MODULE_ALIAS("md-personality-8"); /* RAID6 */
-+MODULE_ALIAS("md-raid6");
-+MODULE_ALIAS("md-level-6");
-+
-+/* This used to be two separate modules, they were: */
-+MODULE_ALIAS("raid5");
-+MODULE_ALIAS("raid6");
diff -Nur linux-4.1.10.orig/drivers/md/raid5.h linux-4.1.10/drivers/md/raid5.h
--- linux-4.1.10.orig/drivers/md/raid5.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/md/raid5.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/md/raid5.h 2015-10-12 22:33:32.252678339 +0200
@@ -495,6 +495,7 @@
int recovery_disabled;
/* per cpu variables */
@@ -14286,621 +6429,9 @@ diff -Nur linux-4.1.10.orig/drivers/md/raid5.h linux-4.1.10/drivers/md/raid5.h
struct page *spare_page; /* Used when checking P/Q in raid6 */
struct flex_array *scribble; /* space for constructing buffer
* lists and performing address
-diff -Nur linux-4.1.10.orig/drivers/md/raid5.h.orig linux-4.1.10/drivers/md/raid5.h.orig
---- linux-4.1.10.orig/drivers/md/raid5.h.orig 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/drivers/md/raid5.h.orig 2015-10-03 13:49:38.000000000 +0200
-@@ -0,0 +1,608 @@
-+#ifndef _RAID5_H
-+#define _RAID5_H
-+
-+#include <linux/raid/xor.h>
-+#include <linux/dmaengine.h>
-+
-+/*
-+ *
-+ * Each stripe contains one buffer per device. Each buffer can be in
-+ * one of a number of states stored in "flags". Changes between
-+ * these states happen *almost* exclusively under the protection of the
-+ * STRIPE_ACTIVE flag. Some very specific changes can happen in bi_end_io, and
-+ * these are not protected by STRIPE_ACTIVE.
-+ *
-+ * The flag bits that are used to represent these states are:
-+ * R5_UPTODATE and R5_LOCKED
-+ *
-+ * State Empty == !UPTODATE, !LOCK
-+ * We have no data, and there is no active request
-+ * State Want == !UPTODATE, LOCK
-+ * A read request is being submitted for this block
-+ * State Dirty == UPTODATE, LOCK
-+ * Some new data is in this buffer, and it is being written out
-+ * State Clean == UPTODATE, !LOCK
-+ * We have valid data which is the same as on disc
-+ *
-+ * The possible state transitions are:
-+ *
-+ * Empty -> Want - on read or write to get old data for parity calc
-+ * Empty -> Dirty - on compute_parity to satisfy write/sync request.
-+ * Empty -> Clean - on compute_block when computing a block for failed drive
-+ * Want -> Empty - on failed read
-+ * Want -> Clean - on successful completion of read request
-+ * Dirty -> Clean - on successful completion of write request
-+ * Dirty -> Clean - on failed write
-+ * Clean -> Dirty - on compute_parity to satisfy write/sync (RECONSTRUCT or RMW)
-+ *
-+ * The Want->Empty, Want->Clean, Dirty->Clean, transitions
-+ * all happen in b_end_io at interrupt time.
-+ * Each sets the Uptodate bit before releasing the Lock bit.
-+ * This leaves one multi-stage transition:
-+ * Want->Dirty->Clean
-+ * This is safe because thinking that a Clean buffer is actually dirty
-+ * will at worst delay some action, and the stripe will be scheduled
-+ * for attention after the transition is complete.
-+ *
-+ * There is one possibility that is not covered by these states. That
-+ * is if one drive has failed and there is a spare being rebuilt. We
-+ * can't distinguish between a clean block that has been generated
-+ * from parity calculations, and a clean block that has been
-+ * successfully written to the spare ( or to parity when resyncing).
-+ * To distinguish these states we have a stripe bit STRIPE_INSYNC that
-+ * is set whenever a write is scheduled to the spare, or to the parity
-+ * disc if there is no spare. A sync request clears this bit, and
-+ * when we find it set with no buffers locked, we know the sync is
-+ * complete.
-+ *
-+ * Buffers for the md device that arrive via make_request are attached
-+ * to the appropriate stripe in one of two lists linked on b_reqnext.
-+ * One list (bh_read) for read requests, one (bh_write) for write.
-+ * There should never be more than one buffer on the two lists
-+ * together, but we are not guaranteed of that so we allow for more.
-+ *
-+ * If a buffer is on the read list when the associated cache buffer is
-+ * Uptodate, the data is copied into the read buffer and it's b_end_io
-+ * routine is called. This may happen in the end_request routine only
-+ * if the buffer has just successfully been read. end_request should
-+ * remove the buffers from the list and then set the Uptodate bit on
-+ * the buffer. Other threads may do this only if they first check
-+ * that the Uptodate bit is set. Once they have checked that they may
-+ * take buffers off the read queue.
-+ *
-+ * When a buffer on the write list is committed for write it is copied
-+ * into the cache buffer, which is then marked dirty, and moved onto a
-+ * third list, the written list (bh_written). Once both the parity
-+ * block and the cached buffer are successfully written, any buffer on
-+ * a written list can be returned with b_end_io.
-+ *
-+ * The write list and read list both act as fifos. The read list,
-+ * write list and written list are protected by the device_lock.
-+ * The device_lock is only for list manipulations and will only be
-+ * held for a very short time. It can be claimed from interrupts.
-+ *
-+ *
-+ * Stripes in the stripe cache can be on one of two lists (or on
-+ * neither). The "inactive_list" contains stripes which are not
-+ * currently being used for any request. They can freely be reused
-+ * for another stripe. The "handle_list" contains stripes that need
-+ * to be handled in some way. Both of these are fifo queues. Each
-+ * stripe is also (potentially) linked to a hash bucket in the hash
-+ * table so that it can be found by sector number. Stripes that are
-+ * not hashed must be on the inactive_list, and will normally be at
-+ * the front. All stripes start life this way.
-+ *
-+ * The inactive_list, handle_list and hash bucket lists are all protected by the
-+ * device_lock.
-+ * - stripes have a reference counter. If count==0, they are on a list.
-+ * - If a stripe might need handling, STRIPE_HANDLE is set.
-+ * - When refcount reaches zero, then if STRIPE_HANDLE it is put on
-+ * handle_list else inactive_list
-+ *
-+ * This, combined with the fact that STRIPE_HANDLE is only ever
-+ * cleared while a stripe has a non-zero count means that if the
-+ * refcount is 0 and STRIPE_HANDLE is set, then it is on the
-+ * handle_list and if recount is 0 and STRIPE_HANDLE is not set, then
-+ * the stripe is on inactive_list.
-+ *
-+ * The possible transitions are:
-+ * activate an unhashed/inactive stripe (get_active_stripe())
-+ * lockdev check-hash unlink-stripe cnt++ clean-stripe hash-stripe unlockdev
-+ * activate a hashed, possibly active stripe (get_active_stripe())
-+ * lockdev check-hash if(!cnt++)unlink-stripe unlockdev
-+ * attach a request to an active stripe (add_stripe_bh())
-+ * lockdev attach-buffer unlockdev
-+ * handle a stripe (handle_stripe())
-+ * setSTRIPE_ACTIVE, clrSTRIPE_HANDLE ...
-+ * (lockdev check-buffers unlockdev) ..
-+ * change-state ..
-+ * record io/ops needed clearSTRIPE_ACTIVE schedule io/ops
-+ * release an active stripe (release_stripe())
-+ * lockdev if (!--cnt) { if STRIPE_HANDLE, add to handle_list else add to inactive-list } unlockdev
-+ *
-+ * The refcount counts each thread that have activated the stripe,
-+ * plus raid5d if it is handling it, plus one for each active request
-+ * on a cached buffer, and plus one if the stripe is undergoing stripe
-+ * operations.
-+ *
-+ * The stripe operations are:
-+ * -copying data between the stripe cache and user application buffers
-+ * -computing blocks to save a disk access, or to recover a missing block
-+ * -updating the parity on a write operation (reconstruct write and
-+ * read-modify-write)
-+ * -checking parity correctness
-+ * -running i/o to disk
-+ * These operations are carried out by raid5_run_ops which uses the async_tx
-+ * api to (optionally) offload operations to dedicated hardware engines.
-+ * When requesting an operation handle_stripe sets the pending bit for the
-+ * operation and increments the count. raid5_run_ops is then run whenever
-+ * the count is non-zero.
-+ * There are some critical dependencies between the operations that prevent some
-+ * from being requested while another is in flight.
-+ * 1/ Parity check operations destroy the in cache version of the parity block,
-+ * so we prevent parity dependent operations like writes and compute_blocks
-+ * from starting while a check is in progress. Some dma engines can perform
-+ * the check without damaging the parity block, in these cases the parity
-+ * block is re-marked up to date (assuming the check was successful) and is
-+ * not re-read from disk.
-+ * 2/ When a write operation is requested we immediately lock the affected
-+ * blocks, and mark them as not up to date. This causes new read requests
-+ * to be held off, as well as parity checks and compute block operations.
-+ * 3/ Once a compute block operation has been requested handle_stripe treats
-+ * that block as if it is up to date. raid5_run_ops guaruntees that any
-+ * operation that is dependent on the compute block result is initiated after
-+ * the compute block completes.
-+ */
-+
-+/*
-+ * Operations state - intermediate states that are visible outside of
-+ * STRIPE_ACTIVE.
-+ * In general _idle indicates nothing is running, _run indicates a data
-+ * processing operation is active, and _result means the data processing result
-+ * is stable and can be acted upon. For simple operations like biofill and
-+ * compute that only have an _idle and _run state they are indicated with
-+ * sh->state flags (STRIPE_BIOFILL_RUN and STRIPE_COMPUTE_RUN)
-+ */
-+/**
-+ * enum check_states - handles syncing / repairing a stripe
-+ * @check_state_idle - check operations are quiesced
-+ * @check_state_run - check operation is running
-+ * @check_state_result - set outside lock when check result is valid
-+ * @check_state_compute_run - check failed and we are repairing
-+ * @check_state_compute_result - set outside lock when compute result is valid
-+ */
-+enum check_states {
-+ check_state_idle = 0,
-+ check_state_run, /* xor parity check */
-+ check_state_run_q, /* q-parity check */
-+ check_state_run_pq, /* pq dual parity check */
-+ check_state_check_result,
-+ check_state_compute_run, /* parity repair */
-+ check_state_compute_result,
-+};
-+
-+/**
-+ * enum reconstruct_states - handles writing or expanding a stripe
-+ */
-+enum reconstruct_states {
-+ reconstruct_state_idle = 0,
-+ reconstruct_state_prexor_drain_run, /* prexor-write */
-+ reconstruct_state_drain_run, /* write */
-+ reconstruct_state_run, /* expand */
-+ reconstruct_state_prexor_drain_result,
-+ reconstruct_state_drain_result,
-+ reconstruct_state_result,
-+};
-+
-+struct stripe_head {
-+ struct hlist_node hash;
-+ struct list_head lru; /* inactive_list or handle_list */
-+ struct llist_node release_list;
-+ struct r5conf *raid_conf;
-+ short generation; /* increments with every
-+ * reshape */
-+ sector_t sector; /* sector of this row */
-+ short pd_idx; /* parity disk index */
-+ short qd_idx; /* 'Q' disk index for raid6 */
-+ short ddf_layout;/* use DDF ordering to calculate Q */
-+ short hash_lock_index;
-+ unsigned long state; /* state flags */
-+ atomic_t count; /* nr of active thread/requests */
-+ int bm_seq; /* sequence number for bitmap flushes */
-+ int disks; /* disks in stripe */
-+ int overwrite_disks; /* total overwrite disks in stripe,
-+ * this is only checked when stripe
-+ * has STRIPE_BATCH_READY
-+ */
-+ enum check_states check_state;
-+ enum reconstruct_states reconstruct_state;
-+ spinlock_t stripe_lock;
-+ int cpu;
-+ struct r5worker_group *group;
-+
-+ struct stripe_head *batch_head; /* protected by stripe lock */
-+ spinlock_t batch_lock; /* only header's lock is useful */
-+ struct list_head batch_list; /* protected by head's batch lock*/
-+ /**
-+ * struct stripe_operations
-+ * @target - STRIPE_OP_COMPUTE_BLK target
-+ * @target2 - 2nd compute target in the raid6 case
-+ * @zero_sum_result - P and Q verification flags
-+ * @request - async service request flags for raid_run_ops
-+ */
-+ struct stripe_operations {
-+ int target, target2;
-+ enum sum_check_flags zero_sum_result;
-+ } ops;
-+ struct r5dev {
-+ /* rreq and rvec are used for the replacement device when
-+ * writing data to both devices.
-+ */
-+ struct bio req, rreq;
-+ struct bio_vec vec, rvec;
-+ struct page *page, *orig_page;
-+ struct bio *toread, *read, *towrite, *written;
-+ sector_t sector; /* sector of this page */
-+ unsigned long flags;
-+ } dev[1]; /* allocated with extra space depending of RAID geometry */
-+};
-+
-+/* stripe_head_state - collects and tracks the dynamic state of a stripe_head
-+ * for handle_stripe.
-+ */
-+struct stripe_head_state {
-+ /* 'syncing' means that we need to read all devices, either
-+ * to check/correct parity, or to reconstruct a missing device.
-+ * 'replacing' means we are replacing one or more drives and
-+ * the source is valid at this point so we don't need to
-+ * read all devices, just the replacement targets.
-+ */
-+ int syncing, expanding, expanded, replacing;
-+ int locked, uptodate, to_read, to_write, failed, written;
-+ int to_fill, compute, req_compute, non_overwrite;
-+ int failed_num[2];
-+ int p_failed, q_failed;
-+ int dec_preread_active;
-+ unsigned long ops_request;
-+
-+ struct bio *return_bi;
-+ struct md_rdev *blocked_rdev;
-+ int handle_bad_blocks;
-+};
-+
-+/* Flags for struct r5dev.flags */
-+enum r5dev_flags {
-+ R5_UPTODATE, /* page contains current data */
-+ R5_LOCKED, /* IO has been submitted on "req" */
-+ R5_DOUBLE_LOCKED,/* Cannot clear R5_LOCKED until 2 writes complete */
-+ R5_OVERWRITE, /* towrite covers whole page */
-+/* and some that are internal to handle_stripe */
-+ R5_Insync, /* rdev && rdev->in_sync at start */
-+ R5_Wantread, /* want to schedule a read */
-+ R5_Wantwrite,
-+ R5_Overlap, /* There is a pending overlapping request
-+ * on this block */
-+ R5_ReadNoMerge, /* prevent bio from merging in block-layer */
-+ R5_ReadError, /* seen a read error here recently */
-+ R5_ReWrite, /* have tried to over-write the readerror */
-+
-+ R5_Expanded, /* This block now has post-expand data */
-+ R5_Wantcompute, /* compute_block in progress treat as
-+ * uptodate
-+ */
-+ R5_Wantfill, /* dev->toread contains a bio that needs
-+ * filling
-+ */
-+ R5_Wantdrain, /* dev->towrite needs to be drained */
-+ R5_WantFUA, /* Write should be FUA */
-+ R5_SyncIO, /* The IO is sync */
-+ R5_WriteError, /* got a write error - need to record it */
-+ R5_MadeGood, /* A bad block has been fixed by writing to it */
-+ R5_ReadRepl, /* Will/did read from replacement rather than orig */
-+ R5_MadeGoodRepl,/* A bad block on the replacement device has been
-+ * fixed by writing to it */
-+ R5_NeedReplace, /* This device has a replacement which is not
-+ * up-to-date at this stripe. */
-+ R5_WantReplace, /* We need to update the replacement, we have read
-+ * data in, and now is a good time to write it out.
-+ */
-+ R5_Discard, /* Discard the stripe */
-+ R5_SkipCopy, /* Don't copy data from bio to stripe cache */
-+};
-+
-+/*
-+ * Stripe state
-+ */
-+enum {
-+ STRIPE_ACTIVE,
-+ STRIPE_HANDLE,
-+ STRIPE_SYNC_REQUESTED,
-+ STRIPE_SYNCING,
-+ STRIPE_INSYNC,
-+ STRIPE_REPLACED,
-+ STRIPE_PREREAD_ACTIVE,
-+ STRIPE_DELAYED,
-+ STRIPE_DEGRADED,
-+ STRIPE_BIT_DELAY,
-+ STRIPE_EXPANDING,
-+ STRIPE_EXPAND_SOURCE,
-+ STRIPE_EXPAND_READY,
-+ STRIPE_IO_STARTED, /* do not count towards 'bypass_count' */
-+ STRIPE_FULL_WRITE, /* all blocks are set to be overwritten */
-+ STRIPE_BIOFILL_RUN,
-+ STRIPE_COMPUTE_RUN,
-+ STRIPE_OPS_REQ_PENDING,
-+ STRIPE_ON_UNPLUG_LIST,
-+ STRIPE_DISCARD,
-+ STRIPE_ON_RELEASE_LIST,
-+ STRIPE_BATCH_READY,
-+ STRIPE_BATCH_ERR,
-+ STRIPE_BITMAP_PENDING, /* Being added to bitmap, don't add
-+ * to batch yet.
-+ */
-+};
-+
-+#define STRIPE_EXPAND_SYNC_FLAGS \
-+ ((1 << STRIPE_EXPAND_SOURCE) |\
-+ (1 << STRIPE_EXPAND_READY) |\
-+ (1 << STRIPE_EXPANDING) |\
-+ (1 << STRIPE_SYNC_REQUESTED))
-+/*
-+ * Operation request flags
-+ */
-+enum {
-+ STRIPE_OP_BIOFILL,
-+ STRIPE_OP_COMPUTE_BLK,
-+ STRIPE_OP_PREXOR,
-+ STRIPE_OP_BIODRAIN,
-+ STRIPE_OP_RECONSTRUCT,
-+ STRIPE_OP_CHECK,
-+};
-+
-+/*
-+ * RAID parity calculation preferences
-+ */
-+enum {
-+ PARITY_DISABLE_RMW = 0,
-+ PARITY_ENABLE_RMW,
-+ PARITY_PREFER_RMW,
-+};
-+
-+/*
-+ * Pages requested from set_syndrome_sources()
-+ */
-+enum {
-+ SYNDROME_SRC_ALL,
-+ SYNDROME_SRC_WANT_DRAIN,
-+ SYNDROME_SRC_WRITTEN,
-+};
-+/*
-+ * Plugging:
-+ *
-+ * To improve write throughput, we need to delay the handling of some
-+ * stripes until there has been a chance that several write requests
-+ * for the one stripe have all been collected.
-+ * In particular, any write request that would require pre-reading
-+ * is put on a "delayed" queue until there are no stripes currently
-+ * in a pre-read phase. Further, if the "delayed" queue is empty when
-+ * a stripe is put on it then we "plug" the queue and do not process it
-+ * until an unplug call is made. (the unplug_io_fn() is called).
-+ *
-+ * When preread is initiated on a stripe, we set PREREAD_ACTIVE and add
-+ * it to the count of prereading stripes.
-+ * When write is initiated, or the stripe refcnt == 0 (just in case) we
-+ * clear the PREREAD_ACTIVE flag and decrement the count
-+ * Whenever the 'handle' queue is empty and the device is not plugged, we
-+ * move any strips from delayed to handle and clear the DELAYED flag and set
-+ * PREREAD_ACTIVE.
-+ * In stripe_handle, if we find pre-reading is necessary, we do it if
-+ * PREREAD_ACTIVE is set, else we set DELAYED which will send it to the delayed queue.
-+ * HANDLE gets cleared if stripe_handle leaves nothing locked.
-+ */
-+
-+struct disk_info {
-+ struct md_rdev *rdev, *replacement;
-+};
-+
-+/* NOTE NR_STRIPE_HASH_LOCKS must remain below 64.
-+ * This is because we sometimes take all the spinlocks
-+ * and creating that much locking depth can cause
-+ * problems.
-+ */
-+#define NR_STRIPE_HASH_LOCKS 8
-+#define STRIPE_HASH_LOCKS_MASK (NR_STRIPE_HASH_LOCKS - 1)
-+
-+struct r5worker {
-+ struct work_struct work;
-+ struct r5worker_group *group;
-+ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
-+ bool working;
-+};
-+
-+struct r5worker_group {
-+ struct list_head handle_list;
-+ struct r5conf *conf;
-+ struct r5worker *workers;
-+ int stripes_cnt;
-+};
-+
-+struct r5conf {
-+ struct hlist_head *stripe_hashtbl;
-+ /* only protect corresponding hash list and inactive_list */
-+ spinlock_t hash_locks[NR_STRIPE_HASH_LOCKS];
-+ struct mddev *mddev;
-+ int chunk_sectors;
-+ int level, algorithm, rmw_level;
-+ int max_degraded;
-+ int raid_disks;
-+ int max_nr_stripes;
-+ int min_nr_stripes;
-+
-+ /* reshape_progress is the leading edge of a 'reshape'
-+ * It has value MaxSector when no reshape is happening
-+ * If delta_disks < 0, it is the last sector we started work on,
-+ * else is it the next sector to work on.
-+ */
-+ sector_t reshape_progress;
-+ /* reshape_safe is the trailing edge of a reshape. We know that
-+ * before (or after) this address, all reshape has completed.
-+ */
-+ sector_t reshape_safe;
-+ int previous_raid_disks;
-+ int prev_chunk_sectors;
-+ int prev_algo;
-+ short generation; /* increments with every reshape */
-+ seqcount_t gen_lock; /* lock against generation changes */
-+ unsigned long reshape_checkpoint; /* Time we last updated
-+ * metadata */
-+ long long min_offset_diff; /* minimum difference between
-+ * data_offset and
-+ * new_data_offset across all
-+ * devices. May be negative,
-+ * but is closest to zero.
-+ */
-+
-+ struct list_head handle_list; /* stripes needing handling */
-+ struct list_head hold_list; /* preread ready stripes */
-+ struct list_head delayed_list; /* stripes that have plugged requests */
-+ struct list_head bitmap_list; /* stripes delaying awaiting bitmap update */
-+ struct bio *retry_read_aligned; /* currently retrying aligned bios */
-+ struct bio *retry_read_aligned_list; /* aligned bios retry list */
-+ atomic_t preread_active_stripes; /* stripes with scheduled io */
-+ atomic_t active_aligned_reads;
-+ atomic_t pending_full_writes; /* full write backlog */
-+ int bypass_count; /* bypassed prereads */
-+ int bypass_threshold; /* preread nice */
-+ int skip_copy; /* Don't copy data from bio to stripe cache */
-+ struct list_head *last_hold; /* detect hold_list promotions */
-+
-+ atomic_t reshape_stripes; /* stripes with pending writes for reshape */
-+ /* unfortunately we need two cache names as we temporarily have
-+ * two caches.
-+ */
-+ int active_name;
-+ char cache_name[2][32];
-+ struct kmem_cache *slab_cache; /* for allocating stripes */
-+ struct mutex cache_size_mutex; /* Protect changes to cache size */
-+
-+ int seq_flush, seq_write;
-+ int quiesce;
-+
-+ int fullsync; /* set to 1 if a full sync is needed,
-+ * (fresh device added).
-+ * Cleared when a sync completes.
-+ */
-+ int recovery_disabled;
-+ /* per cpu variables */
-+ struct raid5_percpu {
-+ struct page *spare_page; /* Used when checking P/Q in raid6 */
-+ struct flex_array *scribble; /* space for constructing buffer
-+ * lists and performing address
-+ * conversions
-+ */
-+ } __percpu *percpu;
-+#ifdef CONFIG_HOTPLUG_CPU
-+ struct notifier_block cpu_notify;
-+#endif
-+
-+ /*
-+ * Free stripes pool
-+ */
-+ atomic_t active_stripes;
-+ struct list_head inactive_list[NR_STRIPE_HASH_LOCKS];
-+ atomic_t empty_inactive_list_nr;
-+ struct llist_head released_stripes;
-+ wait_queue_head_t wait_for_stripe;
-+ wait_queue_head_t wait_for_overlap;
-+ unsigned long cache_state;
-+#define R5_INACTIVE_BLOCKED 1 /* release of inactive stripes blocked,
-+ * waiting for 25% to be free
-+ */
-+#define R5_ALLOC_MORE 2 /* It might help to allocate another
-+ * stripe.
-+ */
-+#define R5_DID_ALLOC 4 /* A stripe was allocated, don't allocate
-+ * more until at least one has been
-+ * released. This avoids flooding
-+ * the cache.
-+ */
-+ struct shrinker shrinker;
-+ int pool_size; /* number of disks in stripeheads in pool */
-+ spinlock_t device_lock;
-+ struct disk_info *disks;
-+
-+ /* When taking over an array from a different personality, we store
-+ * the new thread here until we fully activate the array.
-+ */
-+ struct md_thread *thread;
-+ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
-+ struct r5worker_group *worker_groups;
-+ int group_cnt;
-+ int worker_cnt_per_group;
-+};
-+
-+
-+/*
-+ * Our supported algorithms
-+ */
-+#define ALGORITHM_LEFT_ASYMMETRIC 0 /* Rotating Parity N with Data Restart */
-+#define ALGORITHM_RIGHT_ASYMMETRIC 1 /* Rotating Parity 0 with Data Restart */
-+#define ALGORITHM_LEFT_SYMMETRIC 2 /* Rotating Parity N with Data Continuation */
-+#define ALGORITHM_RIGHT_SYMMETRIC 3 /* Rotating Parity 0 with Data Continuation */
-+
-+/* Define non-rotating (raid4) algorithms. These allow
-+ * conversion of raid4 to raid5.
-+ */
-+#define ALGORITHM_PARITY_0 4 /* P or P,Q are initial devices */
-+#define ALGORITHM_PARITY_N 5 /* P or P,Q are final devices. */
-+
-+/* DDF RAID6 layouts differ from md/raid6 layouts in two ways.
-+ * Firstly, the exact positioning of the parity block is slightly
-+ * different between the 'LEFT_*' modes of md and the "_N_*" modes
-+ * of DDF.
-+ * Secondly, or order of datablocks over which the Q syndrome is computed
-+ * is different.
-+ * Consequently we have different layouts for DDF/raid6 than md/raid6.
-+ * These layouts are from the DDFv1.2 spec.
-+ * Interestingly DDFv1.2-Errata-A does not specify N_CONTINUE but
-+ * leaves RLQ=3 as 'Vendor Specific'
-+ */
-+
-+#define ALGORITHM_ROTATING_ZERO_RESTART 8 /* DDF PRL=6 RLQ=1 */
-+#define ALGORITHM_ROTATING_N_RESTART 9 /* DDF PRL=6 RLQ=2 */
-+#define ALGORITHM_ROTATING_N_CONTINUE 10 /*DDF PRL=6 RLQ=3 */
-+
-+/* For every RAID5 algorithm we define a RAID6 algorithm
-+ * with exactly the same layout for data and parity, and
-+ * with the Q block always on the last device (N-1).
-+ * This allows trivial conversion from RAID5 to RAID6
-+ */
-+#define ALGORITHM_LEFT_ASYMMETRIC_6 16
-+#define ALGORITHM_RIGHT_ASYMMETRIC_6 17
-+#define ALGORITHM_LEFT_SYMMETRIC_6 18
-+#define ALGORITHM_RIGHT_SYMMETRIC_6 19
-+#define ALGORITHM_PARITY_0_6 20
-+#define ALGORITHM_PARITY_N_6 ALGORITHM_PARITY_N
-+
-+static inline int algorithm_valid_raid5(int layout)
-+{
-+ return (layout >= 0) &&
-+ (layout <= 5);
-+}
-+static inline int algorithm_valid_raid6(int layout)
-+{
-+ return (layout >= 0 && layout <= 5)
-+ ||
-+ (layout >= 8 && layout <= 10)
-+ ||
-+ (layout >= 16 && layout <= 20);
-+}
-+
-+static inline int algorithm_is_DDF(int layout)
-+{
-+ return layout >= 8 && layout <= 10;
-+}
-+
-+extern void md_raid5_kick_device(struct r5conf *conf);
-+extern int raid5_set_cache_size(struct mddev *mddev, int size);
-+#endif
diff -Nur linux-4.1.10.orig/drivers/misc/hwlat_detector.c linux-4.1.10/drivers/misc/hwlat_detector.c
--- linux-4.1.10.orig/drivers/misc/hwlat_detector.c 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/drivers/misc/hwlat_detector.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/misc/hwlat_detector.c 2015-10-12 22:33:32.252678339 +0200
@@ -0,0 +1,1240 @@
+/*
+ * hwlat_detector.c - A simple Hardware Latency detector.
@@ -16144,7 +7675,7 @@ diff -Nur linux-4.1.10.orig/drivers/misc/hwlat_detector.c linux-4.1.10/drivers/m
+module_exit(detector_exit);
diff -Nur linux-4.1.10.orig/drivers/misc/Kconfig linux-4.1.10/drivers/misc/Kconfig
--- linux-4.1.10.orig/drivers/misc/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/misc/Kconfig 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/misc/Kconfig 2015-10-12 22:33:32.252678339 +0200
@@ -54,6 +54,7 @@
config ATMEL_TCLIB
bool "Atmel AT32/AT91 Timer/Counter Library"
@@ -16217,7 +7748,7 @@ diff -Nur linux-4.1.10.orig/drivers/misc/Kconfig linux-4.1.10/drivers/misc/Kconf
depends on PCI
diff -Nur linux-4.1.10.orig/drivers/misc/Makefile linux-4.1.10/drivers/misc/Makefile
--- linux-4.1.10.orig/drivers/misc/Makefile 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/misc/Makefile 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/misc/Makefile 2015-10-12 22:33:32.252678339 +0200
@@ -38,6 +38,7 @@
obj-$(CONFIG_HMC6352) += hmc6352.o
obj-y += eeprom/
@@ -16228,7 +7759,7 @@ diff -Nur linux-4.1.10.orig/drivers/misc/Makefile linux-4.1.10/drivers/misc/Make
obj-$(CONFIG_ARM_CHARLCD) += arm-charlcd.o
diff -Nur linux-4.1.10.orig/drivers/mmc/host/mmci.c linux-4.1.10/drivers/mmc/host/mmci.c
--- linux-4.1.10.orig/drivers/mmc/host/mmci.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/mmc/host/mmci.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/mmc/host/mmci.c 2015-10-12 22:33:32.252678339 +0200
@@ -1155,15 +1155,12 @@
struct sg_mapping_iter *sg_miter = &host->sg_miter;
struct variant_data *variant = host->variant;
@@ -16256,7 +7787,7 @@ diff -Nur linux-4.1.10.orig/drivers/mmc/host/mmci.c linux-4.1.10/drivers/mmc/hos
* trigger a PIO interrupt as soon as any data is available.
diff -Nur linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c linux-4.1.10/drivers/net/ethernet/3com/3c59x.c
--- linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/ethernet/3com/3c59x.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/ethernet/3com/3c59x.c 2015-10-12 22:33:32.252678339 +0200
@@ -842,9 +842,9 @@
{
struct vortex_private *vp = netdev_priv(dev);
@@ -16286,7 +7817,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c linux-4.1.10/drive
diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
--- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-12 22:33:32.252678339 +0200
@@ -2213,11 +2213,7 @@
}
@@ -16302,7 +7833,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linu
/* no enough descriptor, just stop queue */
diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c
--- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-12 22:33:32.256678075 +0200
@@ -1880,8 +1880,7 @@
return NETDEV_TX_OK;
}
@@ -16315,7 +7846,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linu
/* no enough descriptor, just stop queue */
diff -Nur linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c
--- linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-12 22:33:32.256678075 +0200
@@ -1664,8 +1664,7 @@
struct cmdQ *q = &sge->cmdQ[qid];
unsigned int credits, pidx, genbit, count, use_sched_skb = 0;
@@ -16328,7 +7859,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.1.10
diff -Nur linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c
--- linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c 2015-10-12 22:33:32.256678075 +0200
@@ -1540,7 +1540,7 @@
if (netif_running(ndev)) {
@@ -16384,7 +7915,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c linux-4.1.1
}
diff -Nur linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c linux-4.1.10/drivers/net/ethernet/neterion/s2io.c
--- linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/ethernet/neterion/s2io.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/ethernet/neterion/s2io.c 2015-10-12 22:33:32.256678075 +0200
@@ -4084,12 +4084,7 @@
[skb->priority & (MAX_TX_FIFOS - 1)];
fifo = &mac_control->fifos[queue];
@@ -16401,7 +7932,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c linux-4.1.10/dr
if (__netif_subqueue_stopped(dev, fifo->fifo_no)) {
diff -Nur linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
--- linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-12 22:33:32.256678075 +0200
@@ -2137,10 +2137,8 @@
struct pch_gbe_tx_ring *tx_ring = adapter->tx_ring;
unsigned long flags;
@@ -16417,7 +7948,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
spin_unlock_irqrestore(&tx_ring->tx_lock, flags);
diff -Nur linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c linux-4.1.10/drivers/net/ethernet/realtek/8139too.c
--- linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/ethernet/realtek/8139too.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/ethernet/realtek/8139too.c 2015-10-12 22:33:32.260677811 +0200
@@ -2229,7 +2229,7 @@
struct rtl8139_private *tp = netdev_priv(dev);
const int irq = tp->pci_dev->irq;
@@ -16429,7 +7960,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c linux-4.1.10/
}
diff -Nur linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c
--- linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c 2015-10-12 22:33:32.260677811 +0200
@@ -1629,13 +1629,8 @@
unsigned long flags;
@@ -16448,7 +7979,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.1.10/dr
BDX_ASSERT(f->m.wptr >= f->m.memsz); /* started with valid wptr */
diff -Nur linux-4.1.10.orig/drivers/net/rionet.c linux-4.1.10/drivers/net/rionet.c
--- linux-4.1.10.orig/drivers/net/rionet.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/rionet.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/rionet.c 2015-10-12 22:33:32.260677811 +0200
@@ -174,11 +174,7 @@
unsigned long flags;
int add_num = 1;
@@ -16464,7 +7995,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/rionet.c linux-4.1.10/drivers/net/rionet
add_num = nets[rnet->mport->id].nact;
diff -Nur linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c
--- linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-12 22:33:32.260677811 +0200
@@ -697,7 +697,7 @@
while (!ctx->done.done && msecs--)
udelay(1000);
@@ -16476,7 +8007,7 @@ diff -Nur linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c linux-4.1
break;
diff -Nur linux-4.1.10.orig/drivers/pci/access.c linux-4.1.10/drivers/pci/access.c
--- linux-4.1.10.orig/drivers/pci/access.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/pci/access.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/pci/access.c 2015-10-12 22:33:32.260677811 +0200
@@ -580,7 +580,7 @@
WARN_ON(!dev->block_cfg_access);
@@ -16486,797 +8017,9 @@ diff -Nur linux-4.1.10.orig/drivers/pci/access.c linux-4.1.10/drivers/pci/access
raw_spin_unlock_irqrestore(&pci_lock, flags);
}
EXPORT_SYMBOL_GPL(pci_cfg_access_unlock);
-diff -Nur linux-4.1.10.orig/drivers/pci/access.c.orig linux-4.1.10/drivers/pci/access.c.orig
---- linux-4.1.10.orig/drivers/pci/access.c.orig 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/drivers/pci/access.c.orig 2015-10-03 13:49:38.000000000 +0200
-@@ -0,0 +1,784 @@
-+#include <linux/delay.h>
-+#include <linux/pci.h>
-+#include <linux/module.h>
-+#include <linux/sched.h>
-+#include <linux/slab.h>
-+#include <linux/ioport.h>
-+#include <linux/wait.h>
-+
-+#include "pci.h"
-+
-+/*
-+ * This interrupt-safe spinlock protects all accesses to PCI
-+ * configuration space.
-+ */
-+
-+DEFINE_RAW_SPINLOCK(pci_lock);
-+
-+/*
-+ * Wrappers for all PCI configuration access functions. They just check
-+ * alignment, do locking and call the low-level functions pointed to
-+ * by pci_dev->ops.
-+ */
-+
-+#define PCI_byte_BAD 0
-+#define PCI_word_BAD (pos & 1)
-+#define PCI_dword_BAD (pos & 3)
-+
-+#define PCI_OP_READ(size,type,len) \
-+int pci_bus_read_config_##size \
-+ (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \
-+{ \
-+ int res; \
-+ unsigned long flags; \
-+ u32 data = 0; \
-+ if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \
-+ raw_spin_lock_irqsave(&pci_lock, flags); \
-+ res = bus->ops->read(bus, devfn, pos, len, &data); \
-+ *value = (type)data; \
-+ raw_spin_unlock_irqrestore(&pci_lock, flags); \
-+ return res; \
-+}
-+
-+#define PCI_OP_WRITE(size,type,len) \
-+int pci_bus_write_config_##size \
-+ (struct pci_bus *bus, unsigned int devfn, int pos, type value) \
-+{ \
-+ int res; \
-+ unsigned long flags; \
-+ if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \
-+ raw_spin_lock_irqsave(&pci_lock, flags); \
-+ res = bus->ops->write(bus, devfn, pos, len, value); \
-+ raw_spin_unlock_irqrestore(&pci_lock, flags); \
-+ return res; \
-+}
-+
-+PCI_OP_READ(byte, u8, 1)
-+PCI_OP_READ(word, u16, 2)
-+PCI_OP_READ(dword, u32, 4)
-+PCI_OP_WRITE(byte, u8, 1)
-+PCI_OP_WRITE(word, u16, 2)
-+PCI_OP_WRITE(dword, u32, 4)
-+
-+EXPORT_SYMBOL(pci_bus_read_config_byte);
-+EXPORT_SYMBOL(pci_bus_read_config_word);
-+EXPORT_SYMBOL(pci_bus_read_config_dword);
-+EXPORT_SYMBOL(pci_bus_write_config_byte);
-+EXPORT_SYMBOL(pci_bus_write_config_word);
-+EXPORT_SYMBOL(pci_bus_write_config_dword);
-+
-+int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn,
-+ int where, int size, u32 *val)
-+{
-+ void __iomem *addr;
-+
-+ addr = bus->ops->map_bus(bus, devfn, where);
-+ if (!addr) {
-+ *val = ~0;
-+ return PCIBIOS_DEVICE_NOT_FOUND;
-+ }
-+
-+ if (size == 1)
-+ *val = readb(addr);
-+ else if (size == 2)
-+ *val = readw(addr);
-+ else
-+ *val = readl(addr);
-+
-+ return PCIBIOS_SUCCESSFUL;
-+}
-+EXPORT_SYMBOL_GPL(pci_generic_config_read);
-+
-+int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn,
-+ int where, int size, u32 val)
-+{
-+ void __iomem *addr;
-+
-+ addr = bus->ops->map_bus(bus, devfn, where);
-+ if (!addr)
-+ return PCIBIOS_DEVICE_NOT_FOUND;
-+
-+ if (size == 1)
-+ writeb(val, addr);
-+ else if (size == 2)
-+ writew(val, addr);
-+ else
-+ writel(val, addr);
-+
-+ return PCIBIOS_SUCCESSFUL;
-+}
-+EXPORT_SYMBOL_GPL(pci_generic_config_write);
-+
-+int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn,
-+ int where, int size, u32 *val)
-+{
-+ void __iomem *addr;
-+
-+ addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
-+ if (!addr) {
-+ *val = ~0;
-+ return PCIBIOS_DEVICE_NOT_FOUND;
-+ }
-+
-+ *val = readl(addr);
-+
-+ if (size <= 2)
-+ *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
-+
-+ return PCIBIOS_SUCCESSFUL;
-+}
-+EXPORT_SYMBOL_GPL(pci_generic_config_read32);
-+
-+int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn,
-+ int where, int size, u32 val)
-+{
-+ void __iomem *addr;
-+ u32 mask, tmp;
-+
-+ addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
-+ if (!addr)
-+ return PCIBIOS_DEVICE_NOT_FOUND;
-+
-+ if (size == 4) {
-+ writel(val, addr);
-+ return PCIBIOS_SUCCESSFUL;
-+ } else {
-+ mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
-+ }
-+
-+ tmp = readl(addr) & mask;
-+ tmp |= val << ((where & 0x3) * 8);
-+ writel(tmp, addr);
-+
-+ return PCIBIOS_SUCCESSFUL;
-+}
-+EXPORT_SYMBOL_GPL(pci_generic_config_write32);
-+
-+/**
-+ * pci_bus_set_ops - Set raw operations of pci bus
-+ * @bus: pci bus struct
-+ * @ops: new raw operations
-+ *
-+ * Return previous raw operations
-+ */
-+struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops)
-+{
-+ struct pci_ops *old_ops;
-+ unsigned long flags;
-+
-+ raw_spin_lock_irqsave(&pci_lock, flags);
-+ old_ops = bus->ops;
-+ bus->ops = ops;
-+ raw_spin_unlock_irqrestore(&pci_lock, flags);
-+ return old_ops;
-+}
-+EXPORT_SYMBOL(pci_bus_set_ops);
-+
-+/**
-+ * pci_read_vpd - Read one entry from Vital Product Data
-+ * @dev: pci device struct
-+ * @pos: offset in vpd space
-+ * @count: number of bytes to read
-+ * @buf: pointer to where to store result
-+ *
-+ */
-+ssize_t pci_read_vpd(struct pci_dev *dev, loff_t pos, size_t count, void *buf)
-+{
-+ if (!dev->vpd || !dev->vpd->ops)
-+ return -ENODEV;
-+ return dev->vpd->ops->read(dev, pos, count, buf);
-+}
-+EXPORT_SYMBOL(pci_read_vpd);
-+
-+/**
-+ * pci_write_vpd - Write entry to Vital Product Data
-+ * @dev: pci device struct
-+ * @pos: offset in vpd space
-+ * @count: number of bytes to write
-+ * @buf: buffer containing write data
-+ *
-+ */
-+ssize_t pci_write_vpd(struct pci_dev *dev, loff_t pos, size_t count, const void *buf)
-+{
-+ if (!dev->vpd || !dev->vpd->ops)
-+ return -ENODEV;
-+ return dev->vpd->ops->write(dev, pos, count, buf);
-+}
-+EXPORT_SYMBOL(pci_write_vpd);
-+
-+/*
-+ * The following routines are to prevent the user from accessing PCI config
-+ * space when it's unsafe to do so. Some devices require this during BIST and
-+ * we're required to prevent it during D-state transitions.
-+ *
-+ * We have a bit per device to indicate it's blocked and a global wait queue
-+ * for callers to sleep on until devices are unblocked.
-+ */
-+static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait);
-+
-+static noinline void pci_wait_cfg(struct pci_dev *dev)
-+{
-+ DECLARE_WAITQUEUE(wait, current);
-+
-+ __add_wait_queue(&pci_cfg_wait, &wait);
-+ do {
-+ set_current_state(TASK_UNINTERRUPTIBLE);
-+ raw_spin_unlock_irq(&pci_lock);
-+ schedule();
-+ raw_spin_lock_irq(&pci_lock);
-+ } while (dev->block_cfg_access);
-+ __remove_wait_queue(&pci_cfg_wait, &wait);
-+}
-+
-+/* Returns 0 on success, negative values indicate error. */
-+#define PCI_USER_READ_CONFIG(size,type) \
-+int pci_user_read_config_##size \
-+ (struct pci_dev *dev, int pos, type *val) \
-+{ \
-+ int ret = PCIBIOS_SUCCESSFUL; \
-+ u32 data = -1; \
-+ if (PCI_##size##_BAD) \
-+ return -EINVAL; \
-+ raw_spin_lock_irq(&pci_lock); \
-+ if (unlikely(dev->block_cfg_access)) \
-+ pci_wait_cfg(dev); \
-+ ret = dev->bus->ops->read(dev->bus, dev->devfn, \
-+ pos, sizeof(type), &data); \
-+ raw_spin_unlock_irq(&pci_lock); \
-+ *val = (type)data; \
-+ return pcibios_err_to_errno(ret); \
-+} \
-+EXPORT_SYMBOL_GPL(pci_user_read_config_##size);
-+
-+/* Returns 0 on success, negative values indicate error. */
-+#define PCI_USER_WRITE_CONFIG(size,type) \
-+int pci_user_write_config_##size \
-+ (struct pci_dev *dev, int pos, type val) \
-+{ \
-+ int ret = PCIBIOS_SUCCESSFUL; \
-+ if (PCI_##size##_BAD) \
-+ return -EINVAL; \
-+ raw_spin_lock_irq(&pci_lock); \
-+ if (unlikely(dev->block_cfg_access)) \
-+ pci_wait_cfg(dev); \
-+ ret = dev->bus->ops->write(dev->bus, dev->devfn, \
-+ pos, sizeof(type), val); \
-+ raw_spin_unlock_irq(&pci_lock); \
-+ return pcibios_err_to_errno(ret); \
-+} \
-+EXPORT_SYMBOL_GPL(pci_user_write_config_##size);
-+
-+PCI_USER_READ_CONFIG(byte, u8)
-+PCI_USER_READ_CONFIG(word, u16)
-+PCI_USER_READ_CONFIG(dword, u32)
-+PCI_USER_WRITE_CONFIG(byte, u8)
-+PCI_USER_WRITE_CONFIG(word, u16)
-+PCI_USER_WRITE_CONFIG(dword, u32)
-+
-+/* VPD access through PCI 2.2+ VPD capability */
-+
-+#define PCI_VPD_PCI22_SIZE (PCI_VPD_ADDR_MASK + 1)
-+
-+struct pci_vpd_pci22 {
-+ struct pci_vpd base;
-+ struct mutex lock;
-+ u16 flag;
-+ bool busy;
-+ u8 cap;
-+};
-+
-+/*
-+ * Wait for last operation to complete.
-+ * This code has to spin since there is no other notification from the PCI
-+ * hardware. Since the VPD is often implemented by serial attachment to an
-+ * EEPROM, it may take many milliseconds to complete.
-+ *
-+ * Returns 0 on success, negative values indicate error.
-+ */
-+static int pci_vpd_pci22_wait(struct pci_dev *dev)
-+{
-+ struct pci_vpd_pci22 *vpd =
-+ container_of(dev->vpd, struct pci_vpd_pci22, base);
-+ unsigned long timeout = jiffies + HZ/20 + 2;
-+ u16 status;
-+ int ret;
-+
-+ if (!vpd->busy)
-+ return 0;
-+
-+ for (;;) {
-+ ret = pci_user_read_config_word(dev, vpd->cap + PCI_VPD_ADDR,
-+ &status);
-+ if (ret < 0)
-+ return ret;
-+
-+ if ((status & PCI_VPD_ADDR_F) == vpd->flag) {
-+ vpd->busy = false;
-+ return 0;
-+ }
-+
-+ if (time_after(jiffies, timeout)) {
-+ dev_printk(KERN_DEBUG, &dev->dev, "vpd r/w failed. This is likely a firmware bug on this device. Contact the card vendor for a firmware update\n");
-+ return -ETIMEDOUT;
-+ }
-+ if (fatal_signal_pending(current))
-+ return -EINTR;
-+ if (!cond_resched())
-+ udelay(10);
-+ }
-+}
-+
-+static ssize_t pci_vpd_pci22_read(struct pci_dev *dev, loff_t pos, size_t count,
-+ void *arg)
-+{
-+ struct pci_vpd_pci22 *vpd =
-+ container_of(dev->vpd, struct pci_vpd_pci22, base);
-+ int ret;
-+ loff_t end = pos + count;
-+ u8 *buf = arg;
-+
-+ if (pos < 0 || pos > vpd->base.len || end > vpd->base.len)
-+ return -EINVAL;
-+
-+ if (mutex_lock_killable(&vpd->lock))
-+ return -EINTR;
-+
-+ ret = pci_vpd_pci22_wait(dev);
-+ if (ret < 0)
-+ goto out;
-+
-+ while (pos < end) {
-+ u32 val;
-+ unsigned int i, skip;
-+
-+ ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR,
-+ pos & ~3);
-+ if (ret < 0)
-+ break;
-+ vpd->busy = true;
-+ vpd->flag = PCI_VPD_ADDR_F;
-+ ret = pci_vpd_pci22_wait(dev);
-+ if (ret < 0)
-+ break;
-+
-+ ret = pci_user_read_config_dword(dev, vpd->cap + PCI_VPD_DATA, &val);
-+ if (ret < 0)
-+ break;
-+
-+ skip = pos & 3;
-+ for (i = 0; i < sizeof(u32); i++) {
-+ if (i >= skip) {
-+ *buf++ = val;
-+ if (++pos == end)
-+ break;
-+ }
-+ val >>= 8;
-+ }
-+ }
-+out:
-+ mutex_unlock(&vpd->lock);
-+ return ret ? ret : count;
-+}
-+
-+static ssize_t pci_vpd_pci22_write(struct pci_dev *dev, loff_t pos, size_t count,
-+ const void *arg)
-+{
-+ struct pci_vpd_pci22 *vpd =
-+ container_of(dev->vpd, struct pci_vpd_pci22, base);
-+ const u8 *buf = arg;
-+ loff_t end = pos + count;
-+ int ret = 0;
-+
-+ if (pos < 0 || (pos & 3) || (count & 3) || end > vpd->base.len)
-+ return -EINVAL;
-+
-+ if (mutex_lock_killable(&vpd->lock))
-+ return -EINTR;
-+
-+ ret = pci_vpd_pci22_wait(dev);
-+ if (ret < 0)
-+ goto out;
-+
-+ while (pos < end) {
-+ u32 val;
-+
-+ val = *buf++;
-+ val |= *buf++ << 8;
-+ val |= *buf++ << 16;
-+ val |= *buf++ << 24;
-+
-+ ret = pci_user_write_config_dword(dev, vpd->cap + PCI_VPD_DATA, val);
-+ if (ret < 0)
-+ break;
-+ ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR,
-+ pos | PCI_VPD_ADDR_F);
-+ if (ret < 0)
-+ break;
-+
-+ vpd->busy = true;
-+ vpd->flag = 0;
-+ ret = pci_vpd_pci22_wait(dev);
-+ if (ret < 0)
-+ break;
-+
-+ pos += sizeof(u32);
-+ }
-+out:
-+ mutex_unlock(&vpd->lock);
-+ return ret ? ret : count;
-+}
-+
-+static void pci_vpd_pci22_release(struct pci_dev *dev)
-+{
-+ kfree(container_of(dev->vpd, struct pci_vpd_pci22, base));
-+}
-+
-+static const struct pci_vpd_ops pci_vpd_pci22_ops = {
-+ .read = pci_vpd_pci22_read,
-+ .write = pci_vpd_pci22_write,
-+ .release = pci_vpd_pci22_release,
-+};
-+
-+static ssize_t pci_vpd_f0_read(struct pci_dev *dev, loff_t pos, size_t count,
-+ void *arg)
-+{
-+ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn));
-+ ssize_t ret;
-+
-+ if (!tdev)
-+ return -ENODEV;
-+
-+ ret = pci_read_vpd(tdev, pos, count, arg);
-+ pci_dev_put(tdev);
-+ return ret;
-+}
-+
-+static ssize_t pci_vpd_f0_write(struct pci_dev *dev, loff_t pos, size_t count,
-+ const void *arg)
-+{
-+ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn));
-+ ssize_t ret;
-+
-+ if (!tdev)
-+ return -ENODEV;
-+
-+ ret = pci_write_vpd(tdev, pos, count, arg);
-+ pci_dev_put(tdev);
-+ return ret;
-+}
-+
-+static const struct pci_vpd_ops pci_vpd_f0_ops = {
-+ .read = pci_vpd_f0_read,
-+ .write = pci_vpd_f0_write,
-+ .release = pci_vpd_pci22_release,
-+};
-+
-+static int pci_vpd_f0_dev_check(struct pci_dev *dev)
-+{
-+ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn));
-+ int ret = 0;
-+
-+ if (!tdev)
-+ return -ENODEV;
-+ if (!tdev->vpd || !tdev->multifunction ||
-+ dev->class != tdev->class || dev->vendor != tdev->vendor ||
-+ dev->device != tdev->device)
-+ ret = -ENODEV;
-+
-+ pci_dev_put(tdev);
-+ return ret;
-+}
-+
-+int pci_vpd_pci22_init(struct pci_dev *dev)
-+{
-+ struct pci_vpd_pci22 *vpd;
-+ u8 cap;
-+
-+ cap = pci_find_capability(dev, PCI_CAP_ID_VPD);
-+ if (!cap)
-+ return -ENODEV;
-+ if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) {
-+ int ret = pci_vpd_f0_dev_check(dev);
-+
-+ if (ret)
-+ return ret;
-+ }
-+ vpd = kzalloc(sizeof(*vpd), GFP_ATOMIC);
-+ if (!vpd)
-+ return -ENOMEM;
-+
-+ vpd->base.len = PCI_VPD_PCI22_SIZE;
-+ if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0)
-+ vpd->base.ops = &pci_vpd_f0_ops;
-+ else
-+ vpd->base.ops = &pci_vpd_pci22_ops;
-+ mutex_init(&vpd->lock);
-+ vpd->cap = cap;
-+ vpd->busy = false;
-+ dev->vpd = &vpd->base;
-+ return 0;
-+}
-+
-+/**
-+ * pci_cfg_access_lock - Lock PCI config reads/writes
-+ * @dev: pci device struct
-+ *
-+ * When access is locked, any userspace reads or writes to config
-+ * space and concurrent lock requests will sleep until access is
-+ * allowed via pci_cfg_access_unlocked again.
-+ */
-+void pci_cfg_access_lock(struct pci_dev *dev)
-+{
-+ might_sleep();
-+
-+ raw_spin_lock_irq(&pci_lock);
-+ if (dev->block_cfg_access)
-+ pci_wait_cfg(dev);
-+ dev->block_cfg_access = 1;
-+ raw_spin_unlock_irq(&pci_lock);
-+}
-+EXPORT_SYMBOL_GPL(pci_cfg_access_lock);
-+
-+/**
-+ * pci_cfg_access_trylock - try to lock PCI config reads/writes
-+ * @dev: pci device struct
-+ *
-+ * Same as pci_cfg_access_lock, but will return 0 if access is
-+ * already locked, 1 otherwise. This function can be used from
-+ * atomic contexts.
-+ */
-+bool pci_cfg_access_trylock(struct pci_dev *dev)
-+{
-+ unsigned long flags;
-+ bool locked = true;
-+
-+ raw_spin_lock_irqsave(&pci_lock, flags);
-+ if (dev->block_cfg_access)
-+ locked = false;
-+ else
-+ dev->block_cfg_access = 1;
-+ raw_spin_unlock_irqrestore(&pci_lock, flags);
-+
-+ return locked;
-+}
-+EXPORT_SYMBOL_GPL(pci_cfg_access_trylock);
-+
-+/**
-+ * pci_cfg_access_unlock - Unlock PCI config reads/writes
-+ * @dev: pci device struct
-+ *
-+ * This function allows PCI config accesses to resume.
-+ */
-+void pci_cfg_access_unlock(struct pci_dev *dev)
-+{
-+ unsigned long flags;
-+
-+ raw_spin_lock_irqsave(&pci_lock, flags);
-+
-+ /* This indicates a problem in the caller, but we don't need
-+ * to kill them, unlike a double-block above. */
-+ WARN_ON(!dev->block_cfg_access);
-+
-+ dev->block_cfg_access = 0;
-+ wake_up_all(&pci_cfg_wait);
-+ raw_spin_unlock_irqrestore(&pci_lock, flags);
-+}
-+EXPORT_SYMBOL_GPL(pci_cfg_access_unlock);
-+
-+static inline int pcie_cap_version(const struct pci_dev *dev)
-+{
-+ return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS;
-+}
-+
-+bool pcie_cap_has_lnkctl(const struct pci_dev *dev)
-+{
-+ int type = pci_pcie_type(dev);
-+
-+ return type == PCI_EXP_TYPE_ENDPOINT ||
-+ type == PCI_EXP_TYPE_LEG_END ||
-+ type == PCI_EXP_TYPE_ROOT_PORT ||
-+ type == PCI_EXP_TYPE_UPSTREAM ||
-+ type == PCI_EXP_TYPE_DOWNSTREAM ||
-+ type == PCI_EXP_TYPE_PCI_BRIDGE ||
-+ type == PCI_EXP_TYPE_PCIE_BRIDGE;
-+}
-+
-+static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev)
-+{
-+ int type = pci_pcie_type(dev);
-+
-+ return (type == PCI_EXP_TYPE_ROOT_PORT ||
-+ type == PCI_EXP_TYPE_DOWNSTREAM) &&
-+ pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT;
-+}
-+
-+static inline bool pcie_cap_has_rtctl(const struct pci_dev *dev)
-+{
-+ int type = pci_pcie_type(dev);
-+
-+ return type == PCI_EXP_TYPE_ROOT_PORT ||
-+ type == PCI_EXP_TYPE_RC_EC;
-+}
-+
-+static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos)
-+{
-+ if (!pci_is_pcie(dev))
-+ return false;
-+
-+ switch (pos) {
-+ case PCI_EXP_FLAGS:
-+ return true;
-+ case PCI_EXP_DEVCAP:
-+ case PCI_EXP_DEVCTL:
-+ case PCI_EXP_DEVSTA:
-+ return true;
-+ case PCI_EXP_LNKCAP:
-+ case PCI_EXP_LNKCTL:
-+ case PCI_EXP_LNKSTA:
-+ return pcie_cap_has_lnkctl(dev);
-+ case PCI_EXP_SLTCAP:
-+ case PCI_EXP_SLTCTL:
-+ case PCI_EXP_SLTSTA:
-+ return pcie_cap_has_sltctl(dev);
-+ case PCI_EXP_RTCTL:
-+ case PCI_EXP_RTCAP:
-+ case PCI_EXP_RTSTA:
-+ return pcie_cap_has_rtctl(dev);
-+ case PCI_EXP_DEVCAP2:
-+ case PCI_EXP_DEVCTL2:
-+ case PCI_EXP_LNKCAP2:
-+ case PCI_EXP_LNKCTL2:
-+ case PCI_EXP_LNKSTA2:
-+ return pcie_cap_version(dev) > 1;
-+ default:
-+ return false;
-+ }
-+}
-+
-+/*
-+ * Note that these accessor functions are only for the "PCI Express
-+ * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the
-+ * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.)
-+ */
-+int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val)
-+{
-+ int ret;
-+
-+ *val = 0;
-+ if (pos & 1)
-+ return -EINVAL;
-+
-+ if (pcie_capability_reg_implemented(dev, pos)) {
-+ ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val);
-+ /*
-+ * Reset *val to 0 if pci_read_config_word() fails, it may
-+ * have been written as 0xFFFF if hardware error happens
-+ * during pci_read_config_word().
-+ */
-+ if (ret)
-+ *val = 0;
-+ return ret;
-+ }
-+
-+ /*
-+ * For Functions that do not implement the Slot Capabilities,
-+ * Slot Status, and Slot Control registers, these spaces must
-+ * be hardwired to 0b, with the exception of the Presence Detect
-+ * State bit in the Slot Status register of Downstream Ports,
-+ * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8)
-+ */
-+ if (pci_is_pcie(dev) && pos == PCI_EXP_SLTSTA &&
-+ pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) {
-+ *val = PCI_EXP_SLTSTA_PDS;
-+ }
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(pcie_capability_read_word);
-+
-+int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val)
-+{
-+ int ret;
-+
-+ *val = 0;
-+ if (pos & 3)
-+ return -EINVAL;
-+
-+ if (pcie_capability_reg_implemented(dev, pos)) {
-+ ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val);
-+ /*
-+ * Reset *val to 0 if pci_read_config_dword() fails, it may
-+ * have been written as 0xFFFFFFFF if hardware error happens
-+ * during pci_read_config_dword().
-+ */
-+ if (ret)
-+ *val = 0;
-+ return ret;
-+ }
-+
-+ if (pci_is_pcie(dev) && pos == PCI_EXP_SLTCTL &&
-+ pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) {
-+ *val = PCI_EXP_SLTSTA_PDS;
-+ }
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(pcie_capability_read_dword);
-+
-+int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val)
-+{
-+ if (pos & 1)
-+ return -EINVAL;
-+
-+ if (!pcie_capability_reg_implemented(dev, pos))
-+ return 0;
-+
-+ return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val);
-+}
-+EXPORT_SYMBOL(pcie_capability_write_word);
-+
-+int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val)
-+{
-+ if (pos & 3)
-+ return -EINVAL;
-+
-+ if (!pcie_capability_reg_implemented(dev, pos))
-+ return 0;
-+
-+ return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val);
-+}
-+EXPORT_SYMBOL(pcie_capability_write_dword);
-+
-+int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos,
-+ u16 clear, u16 set)
-+{
-+ int ret;
-+ u16 val;
-+
-+ ret = pcie_capability_read_word(dev, pos, &val);
-+ if (!ret) {
-+ val &= ~clear;
-+ val |= set;
-+ ret = pcie_capability_write_word(dev, pos, val);
-+ }
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL(pcie_capability_clear_and_set_word);
-+
-+int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos,
-+ u32 clear, u32 set)
-+{
-+ int ret;
-+ u32 val;
-+
-+ ret = pcie_capability_read_dword(dev, pos, &val);
-+ if (!ret) {
-+ val &= ~clear;
-+ val |= set;
-+ ret = pcie_capability_write_dword(dev, pos, val);
-+ }
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL(pcie_capability_clear_and_set_dword);
diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c linux-4.1.10/drivers/scsi/fcoe/fcoe.c
--- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/scsi/fcoe/fcoe.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/scsi/fcoe/fcoe.c 2015-10-12 22:33:32.260677811 +0200
@@ -1287,7 +1287,7 @@
struct sk_buff *skb;
#ifdef CONFIG_SMP
@@ -17342,7 +8085,7 @@ diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c linux-4.1.10/drivers/scsi/f
diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c
--- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-12 22:33:32.260677811 +0200
@@ -831,7 +831,7 @@
INIT_LIST_HEAD(&del_list);
@@ -17363,7 +8106,7 @@ diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.1.10/drivers/s
/* Removes fcf from current list */
diff -Nur linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c linux-4.1.10/drivers/scsi/libfc/fc_exch.c
--- linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/scsi/libfc/fc_exch.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/scsi/libfc/fc_exch.c 2015-10-12 22:33:32.260677811 +0200
@@ -814,10 +814,10 @@
}
memset(ep, 0, sizeof(*ep));
@@ -17379,7 +8122,7 @@ diff -Nur linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c linux-4.1.10/drivers/sc
if (pool->left != FC_XID_UNKNOWN) {
diff -Nur linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c linux-4.1.10/drivers/scsi/libsas/sas_ata.c
--- linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/scsi/libsas/sas_ata.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/scsi/libsas/sas_ata.c 2015-10-12 22:33:32.264677547 +0200
@@ -190,7 +190,7 @@
/* TODO: audit callers to ensure they are ready for qc_issue to
* unconditionally re-enable interrupts
@@ -17400,7 +8143,7 @@ diff -Nur linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c linux-4.1.10/drivers/s
diff -Nur linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h
--- linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h 2015-10-12 22:33:32.264677547 +0200
@@ -59,12 +59,12 @@
{
unsigned long flags;
@@ -17418,7 +8161,7 @@ diff -Nur linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.1.10/drive
static inline uint8_t *
diff -Nur linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c
--- linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-12 22:33:32.264677547 +0200
@@ -29,6 +29,7 @@
#include <linux/pm.h>
#include <linux/thermal.h>
@@ -17523,7 +8266,7 @@ diff -Nur linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.1.10/
&per_cpu(pkg_temp_thermal_threshold_work, i));
diff -Nur linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c linux-4.1.10/drivers/tty/serial/8250/8250_core.c
--- linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/tty/serial/8250/8250_core.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/tty/serial/8250/8250_core.c 2015-10-12 22:33:32.264677547 +0200
@@ -36,6 +36,7 @@
#include <linux/nmi.h>
#include <linux/mutex.h>
@@ -17561,7 +8304,7 @@ diff -Nur linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c linux-4.1.10/dri
spin_lock_irqsave(&port->lock, flags);
diff -Nur linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c linux-4.1.10/drivers/tty/serial/amba-pl011.c
--- linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/tty/serial/amba-pl011.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/tty/serial/amba-pl011.c 2015-10-12 22:33:32.264677547 +0200
@@ -2000,13 +2000,19 @@
clk_enable(uap->clk);
@@ -17597,7 +8340,7 @@ diff -Nur linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c linux-4.1.10/drivers
}
diff -Nur linux-4.1.10.orig/drivers/tty/serial/omap-serial.c linux-4.1.10/drivers/tty/serial/omap-serial.c
--- linux-4.1.10.orig/drivers/tty/serial/omap-serial.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/tty/serial/omap-serial.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/tty/serial/omap-serial.c 2015-10-12 22:33:32.264677547 +0200
@@ -1282,13 +1282,10 @@
pm_runtime_get_sync(up->dev);
@@ -17627,7 +8370,7 @@ diff -Nur linux-4.1.10.orig/drivers/tty/serial/omap-serial.c linux-4.1.10/driver
static int __init
diff -Nur linux-4.1.10.orig/drivers/usb/core/hcd.c linux-4.1.10/drivers/usb/core/hcd.c
--- linux-4.1.10.orig/drivers/usb/core/hcd.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/usb/core/hcd.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/usb/core/hcd.c 2015-10-12 22:33:32.264677547 +0200
@@ -1684,9 +1684,9 @@
* and no one may trigger the above deadlock situation when
* running complete() in tasklet.
@@ -17642,7 +8385,7 @@ diff -Nur linux-4.1.10.orig/drivers/usb/core/hcd.c linux-4.1.10/drivers/usb/core
atomic_dec(&urb->use_count);
diff -Nur linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c linux-4.1.10/drivers/usb/gadget/function/f_fs.c
--- linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/usb/gadget/function/f_fs.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/usb/gadget/function/f_fs.c 2015-10-12 22:33:32.264677547 +0200
@@ -1405,7 +1405,7 @@
pr_info("%s(): freeing\n", __func__);
ffs_data_clear(ffs);
@@ -17654,7 +8397,7 @@ diff -Nur linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c linux-4.1.10/driv
}
diff -Nur linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c linux-4.1.10/drivers/usb/gadget/legacy/inode.c
--- linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/drivers/usb/gadget/legacy/inode.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/drivers/usb/gadget/legacy/inode.c 2015-10-12 22:33:32.264677547 +0200
@@ -345,7 +345,7 @@
spin_unlock_irq (&epdata->dev->lock);
@@ -17675,7 +8418,7 @@ diff -Nur linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c linux-4.1.10/drive
} else {
diff -Nur linux-4.1.10.orig/fs/aio.c linux-4.1.10/fs/aio.c
--- linux-4.1.10.orig/fs/aio.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/aio.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/aio.c 2015-10-12 22:33:32.264677547 +0200
@@ -40,6 +40,7 @@
#include <linux/ramfs.h>
#include <linux/percpu-refcount.h>
@@ -17753,7 +8496,7 @@ diff -Nur linux-4.1.10.orig/fs/aio.c linux-4.1.10/fs/aio.c
unsigned i, new_nr;
diff -Nur linux-4.1.10.orig/fs/autofs4/autofs_i.h linux-4.1.10/fs/autofs4/autofs_i.h
--- linux-4.1.10.orig/fs/autofs4/autofs_i.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/autofs4/autofs_i.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/autofs4/autofs_i.h 2015-10-12 22:33:32.268677282 +0200
@@ -34,6 +34,7 @@
#include <linux/sched.h>
#include <linux/mount.h>
@@ -17764,7 +8507,7 @@ diff -Nur linux-4.1.10.orig/fs/autofs4/autofs_i.h linux-4.1.10/fs/autofs4/autofs
diff -Nur linux-4.1.10.orig/fs/autofs4/expire.c linux-4.1.10/fs/autofs4/expire.c
--- linux-4.1.10.orig/fs/autofs4/expire.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/autofs4/expire.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/autofs4/expire.c 2015-10-12 22:33:32.268677282 +0200
@@ -150,7 +150,7 @@
parent = p->d_parent;
if (!spin_trylock(&parent->d_lock)) {
@@ -17776,7 +8519,7 @@ diff -Nur linux-4.1.10.orig/fs/autofs4/expire.c linux-4.1.10/fs/autofs4/expire.c
spin_unlock(&p->d_lock);
diff -Nur linux-4.1.10.orig/fs/buffer.c linux-4.1.10/fs/buffer.c
--- linux-4.1.10.orig/fs/buffer.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/buffer.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/buffer.c 2015-10-12 22:33:32.268677282 +0200
@@ -301,8 +301,7 @@
* decide that the page is now completely done.
*/
@@ -17846,7 +8589,7 @@ diff -Nur linux-4.1.10.orig/fs/buffer.c linux-4.1.10/fs/buffer.c
recalc_bh_state();
diff -Nur linux-4.1.10.orig/fs/dcache.c linux-4.1.10/fs/dcache.c
--- linux-4.1.10.orig/fs/dcache.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/dcache.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/dcache.c 2015-10-12 22:33:32.268677282 +0200
@@ -19,6 +19,7 @@
#include <linux/mm.h>
#include <linux/fs.h>
@@ -17875,7 +8618,7 @@ diff -Nur linux-4.1.10.orig/fs/dcache.c linux-4.1.10/fs/dcache.c
dentry->d_flags &= ~DCACHE_CANT_MOUNT;
diff -Nur linux-4.1.10.orig/fs/eventpoll.c linux-4.1.10/fs/eventpoll.c
--- linux-4.1.10.orig/fs/eventpoll.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/eventpoll.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/eventpoll.c 2015-10-12 22:33:32.268677282 +0200
@@ -505,12 +505,12 @@
*/
static void ep_poll_safewake(wait_queue_head_t *wq)
@@ -17893,7 +8636,7 @@ diff -Nur linux-4.1.10.orig/fs/eventpoll.c linux-4.1.10/fs/eventpoll.c
static void ep_remove_wait_queue(struct eppoll_entry *pwq)
diff -Nur linux-4.1.10.orig/fs/exec.c linux-4.1.10/fs/exec.c
--- linux-4.1.10.orig/fs/exec.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/exec.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/exec.c 2015-10-12 22:33:32.268677282 +0200
@@ -859,12 +859,14 @@
}
}
@@ -17911,7 +8654,7 @@ diff -Nur linux-4.1.10.orig/fs/exec.c linux-4.1.10/fs/exec.c
up_read(&old_mm->mmap_sem);
diff -Nur linux-4.1.10.orig/fs/jbd/checkpoint.c linux-4.1.10/fs/jbd/checkpoint.c
--- linux-4.1.10.orig/fs/jbd/checkpoint.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/jbd/checkpoint.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/jbd/checkpoint.c 2015-10-12 22:33:32.268677282 +0200
@@ -129,6 +129,8 @@
if (journal->j_flags & JFS_ABORT)
return;
@@ -17923,7 +8666,7 @@ diff -Nur linux-4.1.10.orig/fs/jbd/checkpoint.c linux-4.1.10/fs/jbd/checkpoint.c
/*
diff -Nur linux-4.1.10.orig/fs/jbd2/checkpoint.c linux-4.1.10/fs/jbd2/checkpoint.c
--- linux-4.1.10.orig/fs/jbd2/checkpoint.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/jbd2/checkpoint.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/jbd2/checkpoint.c 2015-10-12 22:33:32.268677282 +0200
@@ -116,6 +116,8 @@
nblocks = jbd2_space_needed(journal);
while (jbd2_log_space_left(journal) < nblocks) {
@@ -17935,7 +8678,7 @@ diff -Nur linux-4.1.10.orig/fs/jbd2/checkpoint.c linux-4.1.10/fs/jbd2/checkpoint
/*
diff -Nur linux-4.1.10.orig/fs/namespace.c linux-4.1.10/fs/namespace.c
--- linux-4.1.10.orig/fs/namespace.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/namespace.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/namespace.c 2015-10-12 22:33:32.268677282 +0200
@@ -14,6 +14,7 @@
#include <linux/mnt_namespace.h>
#include <linux/user_namespace.h>
@@ -17960,7 +8703,7 @@ diff -Nur linux-4.1.10.orig/fs/namespace.c linux-4.1.10/fs/namespace.c
* be set to match its requirements. So we must not load that until
diff -Nur linux-4.1.10.orig/fs/ntfs/aops.c linux-4.1.10/fs/ntfs/aops.c
--- linux-4.1.10.orig/fs/ntfs/aops.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/ntfs/aops.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/ntfs/aops.c 2015-10-12 22:33:32.268677282 +0200
@@ -107,8 +107,7 @@
"0x%llx.", (unsigned long long)bh->b_blocknr);
}
@@ -18010,7 +8753,7 @@ diff -Nur linux-4.1.10.orig/fs/ntfs/aops.c linux-4.1.10/fs/ntfs/aops.c
/**
diff -Nur linux-4.1.10.orig/fs/timerfd.c linux-4.1.10/fs/timerfd.c
--- linux-4.1.10.orig/fs/timerfd.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/timerfd.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/timerfd.c 2015-10-12 22:33:32.268677282 +0200
@@ -450,7 +450,10 @@
break;
}
@@ -18025,7 +8768,7 @@ diff -Nur linux-4.1.10.orig/fs/timerfd.c linux-4.1.10/fs/timerfd.c
/*
diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.c linux-4.1.10/fs/xfs/xfs_inode.c
--- linux-4.1.10.orig/fs/xfs/xfs_inode.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/xfs/xfs_inode.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/xfs/xfs_inode.c 2015-10-12 22:33:32.272677018 +0200
@@ -164,7 +164,7 @@
(XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL));
ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) !=
@@ -18150,7 +8893,7 @@ diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.c linux-4.1.10/fs/xfs/xfs_inode.c
i = 0;
diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.h linux-4.1.10/fs/xfs/xfs_inode.h
--- linux-4.1.10.orig/fs/xfs/xfs_inode.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/fs/xfs/xfs_inode.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/fs/xfs/xfs_inode.h 2015-10-12 22:33:32.272677018 +0200
@@ -284,9 +284,9 @@
* Flags for lockdep annotations.
*
@@ -18250,7 +8993,7 @@ diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.h linux-4.1.10/fs/xfs/xfs_inode.h
diff -Nur linux-4.1.10.orig/include/acpi/platform/aclinux.h linux-4.1.10/include/acpi/platform/aclinux.h
--- linux-4.1.10.orig/include/acpi/platform/aclinux.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/acpi/platform/aclinux.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/acpi/platform/aclinux.h 2015-10-12 22:33:32.272677018 +0200
@@ -123,6 +123,7 @@
#define acpi_cache_t struct kmem_cache
@@ -18282,7 +9025,7 @@ diff -Nur linux-4.1.10.orig/include/acpi/platform/aclinux.h linux-4.1.10/include
*/
diff -Nur linux-4.1.10.orig/include/asm-generic/bug.h linux-4.1.10/include/asm-generic/bug.h
--- linux-4.1.10.orig/include/asm-generic/bug.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/asm-generic/bug.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/asm-generic/bug.h 2015-10-12 22:33:32.272677018 +0200
@@ -206,6 +206,20 @@
# define WARN_ON_SMP(x) ({0;})
#endif
@@ -18306,7 +9049,7 @@ diff -Nur linux-4.1.10.orig/include/asm-generic/bug.h linux-4.1.10/include/asm-g
#endif
diff -Nur linux-4.1.10.orig/include/asm-generic/futex.h linux-4.1.10/include/asm-generic/futex.h
--- linux-4.1.10.orig/include/asm-generic/futex.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/asm-generic/futex.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/asm-generic/futex.h 2015-10-12 22:33:32.272677018 +0200
@@ -8,8 +8,7 @@
#ifndef CONFIG_SMP
/*
@@ -18351,7 +9094,7 @@ diff -Nur linux-4.1.10.orig/include/asm-generic/futex.h linux-4.1.10/include/asm
}
diff -Nur linux-4.1.10.orig/include/linux/blkdev.h linux-4.1.10/include/linux/blkdev.h
--- linux-4.1.10.orig/include/linux/blkdev.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/blkdev.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/blkdev.h 2015-10-12 22:33:32.272677018 +0200
@@ -101,6 +101,7 @@
struct list_head queuelist;
union {
@@ -18371,7 +9114,7 @@ diff -Nur linux-4.1.10.orig/include/linux/blkdev.h linux-4.1.10/include/linux/bl
diff -Nur linux-4.1.10.orig/include/linux/blk-mq.h linux-4.1.10/include/linux/blk-mq.h
--- linux-4.1.10.orig/include/linux/blk-mq.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/blk-mq.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/blk-mq.h 2015-10-12 22:33:32.272677018 +0200
@@ -202,6 +202,7 @@
struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index);
@@ -18382,7 +9125,7 @@ diff -Nur linux-4.1.10.orig/include/linux/blk-mq.h linux-4.1.10/include/linux/bl
void blk_mq_start_request(struct request *rq);
diff -Nur linux-4.1.10.orig/include/linux/bottom_half.h linux-4.1.10/include/linux/bottom_half.h
--- linux-4.1.10.orig/include/linux/bottom_half.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/bottom_half.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/bottom_half.h 2015-10-12 22:33:32.272677018 +0200
@@ -4,6 +4,17 @@
#include <linux/preempt.h>
#include <linux/preempt_mask.h>
@@ -18410,7 +9153,7 @@ diff -Nur linux-4.1.10.orig/include/linux/bottom_half.h linux-4.1.10/include/lin
#endif /* _LINUX_BH_H */
diff -Nur linux-4.1.10.orig/include/linux/buffer_head.h linux-4.1.10/include/linux/buffer_head.h
--- linux-4.1.10.orig/include/linux/buffer_head.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/buffer_head.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/buffer_head.h 2015-10-12 22:33:32.272677018 +0200
@@ -75,8 +75,52 @@
struct address_space *b_assoc_map; /* mapping this buffer is
associated with */
@@ -18466,7 +9209,7 @@ diff -Nur linux-4.1.10.orig/include/linux/buffer_head.h linux-4.1.10/include/lin
* and buffer_foo() functions.
diff -Nur linux-4.1.10.orig/include/linux/cgroup.h linux-4.1.10/include/linux/cgroup.h
--- linux-4.1.10.orig/include/linux/cgroup.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/cgroup.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/cgroup.h 2015-10-12 22:33:32.272677018 +0200
@@ -22,6 +22,7 @@
#include <linux/seq_file.h>
#include <linux/kernfs.h>
@@ -18485,7 +9228,7 @@ diff -Nur linux-4.1.10.orig/include/linux/cgroup.h linux-4.1.10/include/linux/cg
/* bits in struct cgroup_subsys_state flags field */
diff -Nur linux-4.1.10.orig/include/linux/completion.h linux-4.1.10/include/linux/completion.h
--- linux-4.1.10.orig/include/linux/completion.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/completion.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/completion.h 2015-10-12 22:33:32.272677018 +0200
@@ -7,8 +7,7 @@
* Atomic wait-for-completion handler data structures.
* See kernel/sched/completion.c for details.
@@ -18521,7 +9264,7 @@ diff -Nur linux-4.1.10.orig/include/linux/completion.h linux-4.1.10/include/linu
/**
diff -Nur linux-4.1.10.orig/include/linux/cpu.h linux-4.1.10/include/linux/cpu.h
--- linux-4.1.10.orig/include/linux/cpu.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/cpu.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/cpu.h 2015-10-12 22:33:32.272677018 +0200
@@ -231,6 +231,8 @@
extern void put_online_cpus(void);
extern void cpu_hotplug_disable(void);
@@ -18542,7 +9285,7 @@ diff -Nur linux-4.1.10.orig/include/linux/cpu.h linux-4.1.10/include/linux/cpu.h
/* These aren't inline functions due to a GCC bug. */
diff -Nur linux-4.1.10.orig/include/linux/delay.h linux-4.1.10/include/linux/delay.h
--- linux-4.1.10.orig/include/linux/delay.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/delay.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/delay.h 2015-10-12 22:33:32.272677018 +0200
@@ -52,4 +52,10 @@
msleep(seconds * 1000);
}
@@ -18556,7 +9299,7 @@ diff -Nur linux-4.1.10.orig/include/linux/delay.h linux-4.1.10/include/linux/del
#endif /* defined(_LINUX_DELAY_H) */
diff -Nur linux-4.1.10.orig/include/linux/ftrace_event.h linux-4.1.10/include/linux/ftrace_event.h
--- linux-4.1.10.orig/include/linux/ftrace_event.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/ftrace_event.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/ftrace_event.h 2015-10-12 22:33:32.272677018 +0200
@@ -66,6 +66,9 @@
unsigned char flags;
unsigned char preempt_count;
@@ -18569,7 +9312,7 @@ diff -Nur linux-4.1.10.orig/include/linux/ftrace_event.h linux-4.1.10/include/li
#define FTRACE_MAX_EVENT \
diff -Nur linux-4.1.10.orig/include/linux/highmem.h linux-4.1.10/include/linux/highmem.h
--- linux-4.1.10.orig/include/linux/highmem.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/highmem.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/highmem.h 2015-10-12 22:33:32.272677018 +0200
@@ -7,6 +7,7 @@
#include <linux/mm.h>
#include <linux/uaccess.h>
@@ -18652,7 +9395,7 @@ diff -Nur linux-4.1.10.orig/include/linux/highmem.h linux-4.1.10/include/linux/h
diff -Nur linux-4.1.10.orig/include/linux/hrtimer.h linux-4.1.10/include/linux/hrtimer.h
--- linux-4.1.10.orig/include/linux/hrtimer.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/hrtimer.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/hrtimer.h 2015-10-12 22:33:32.272677018 +0200
@@ -111,6 +111,11 @@
enum hrtimer_restart (*function)(struct hrtimer *);
struct hrtimer_clock_base *base;
@@ -18699,7 +9442,7 @@ diff -Nur linux-4.1.10.orig/include/linux/hrtimer.h linux-4.1.10/include/linux/h
extern int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp);
diff -Nur linux-4.1.10.orig/include/linux/idr.h linux-4.1.10/include/linux/idr.h
--- linux-4.1.10.orig/include/linux/idr.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/idr.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/idr.h 2015-10-12 22:33:32.272677018 +0200
@@ -95,10 +95,14 @@
* Each idr_preload() should be matched with an invocation of this
* function. See idr_preload() for details.
@@ -18717,7 +9460,7 @@ diff -Nur linux-4.1.10.orig/include/linux/idr.h linux-4.1.10/include/linux/idr.h
* idr_find - return pointer for given id
diff -Nur linux-4.1.10.orig/include/linux/init_task.h linux-4.1.10/include/linux/init_task.h
--- linux-4.1.10.orig/include/linux/init_task.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/init_task.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/init_task.h 2015-10-12 22:33:32.272677018 +0200
@@ -147,9 +147,16 @@
# define INIT_PERF_EVENTS(tsk)
#endif
@@ -18746,7 +9489,7 @@ diff -Nur linux-4.1.10.orig/include/linux/init_task.h linux-4.1.10/include/linux
[PIDTYPE_PGID] = INIT_PID_LINK(PIDTYPE_PGID), \
diff -Nur linux-4.1.10.orig/include/linux/interrupt.h linux-4.1.10/include/linux/interrupt.h
--- linux-4.1.10.orig/include/linux/interrupt.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/interrupt.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/interrupt.h 2015-10-12 22:33:32.272677018 +0200
@@ -61,6 +61,7 @@
* interrupt handler after suspending interrupts. For system
* wakeup devices users need to implement wakeup detection in
@@ -18925,7 +9668,7 @@ diff -Nur linux-4.1.10.orig/include/linux/interrupt.h linux-4.1.10/include/linux
*
diff -Nur linux-4.1.10.orig/include/linux/io-mapping.h linux-4.1.10/include/linux/io-mapping.h
--- linux-4.1.10.orig/include/linux/io-mapping.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/io-mapping.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/io-mapping.h 2015-10-12 22:33:32.272677018 +0200
@@ -141,6 +141,7 @@
io_mapping_map_atomic_wc(struct io_mapping *mapping,
unsigned long offset)
@@ -18944,7 +9687,7 @@ diff -Nur linux-4.1.10.orig/include/linux/io-mapping.h linux-4.1.10/include/linu
/* Non-atomic map/unmap */
diff -Nur linux-4.1.10.orig/include/linux/irqdesc.h linux-4.1.10/include/linux/irqdesc.h
--- linux-4.1.10.orig/include/linux/irqdesc.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/irqdesc.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/irqdesc.h 2015-10-12 22:33:32.272677018 +0200
@@ -63,6 +63,7 @@
unsigned int irqs_unhandled;
atomic_t threads_handled;
@@ -18955,7 +9698,7 @@ diff -Nur linux-4.1.10.orig/include/linux/irqdesc.h linux-4.1.10/include/linux/i
#ifdef CONFIG_SMP
diff -Nur linux-4.1.10.orig/include/linux/irqflags.h linux-4.1.10/include/linux/irqflags.h
--- linux-4.1.10.orig/include/linux/irqflags.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/irqflags.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/irqflags.h 2015-10-12 22:33:32.272677018 +0200
@@ -25,8 +25,6 @@
# define trace_softirqs_enabled(p) ((p)->softirqs_enabled)
# define trace_hardirq_enter() do { current->hardirq_context++; } while (0)
@@ -19008,7 +9751,7 @@ diff -Nur linux-4.1.10.orig/include/linux/irqflags.h linux-4.1.10/include/linux/
#endif
diff -Nur linux-4.1.10.orig/include/linux/irq.h linux-4.1.10/include/linux/irq.h
--- linux-4.1.10.orig/include/linux/irq.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/irq.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/irq.h 2015-10-12 22:33:32.276676754 +0200
@@ -72,6 +72,7 @@
* IRQ_IS_POLLED - Always polled by another interrupt. Exclude
* it from the spurious interrupt detection
@@ -19035,7 +9778,7 @@ diff -Nur linux-4.1.10.orig/include/linux/irq.h linux-4.1.10/include/linux/irq.h
diff -Nur linux-4.1.10.orig/include/linux/irq_work.h linux-4.1.10/include/linux/irq_work.h
--- linux-4.1.10.orig/include/linux/irq_work.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/irq_work.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/irq_work.h 2015-10-12 22:33:32.276676754 +0200
@@ -16,6 +16,7 @@
#define IRQ_WORK_BUSY 2UL
#define IRQ_WORK_FLAGS 3UL
@@ -19046,7 +9789,7 @@ diff -Nur linux-4.1.10.orig/include/linux/irq_work.h linux-4.1.10/include/linux/
unsigned long flags;
diff -Nur linux-4.1.10.orig/include/linux/jbd_common.h linux-4.1.10/include/linux/jbd_common.h
--- linux-4.1.10.orig/include/linux/jbd_common.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/jbd_common.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/jbd_common.h 2015-10-12 22:33:32.276676754 +0200
@@ -15,32 +15,56 @@
static inline void jbd_lock_bh_state(struct buffer_head *bh)
@@ -19106,7 +9849,7 @@ diff -Nur linux-4.1.10.orig/include/linux/jbd_common.h linux-4.1.10/include/linu
#endif
diff -Nur linux-4.1.10.orig/include/linux/kdb.h linux-4.1.10/include/linux/kdb.h
--- linux-4.1.10.orig/include/linux/kdb.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/kdb.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/kdb.h 2015-10-12 22:33:32.276676754 +0200
@@ -167,6 +167,7 @@
extern __printf(1, 2) int kdb_printf(const char *, ...);
typedef __printf(1, 2) int (*kdb_printf_t)(const char *, ...);
@@ -19125,7 +9868,7 @@ diff -Nur linux-4.1.10.orig/include/linux/kdb.h linux-4.1.10/include/linux/kdb.h
char *help, short minlen) { return 0; }
diff -Nur linux-4.1.10.orig/include/linux/kernel.h linux-4.1.10/include/linux/kernel.h
--- linux-4.1.10.orig/include/linux/kernel.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/kernel.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/kernel.h 2015-10-12 22:33:32.276676754 +0200
@@ -188,6 +188,9 @@
*/
# define might_sleep() \
@@ -19164,7 +9907,7 @@ diff -Nur linux-4.1.10.orig/include/linux/kernel.h linux-4.1.10/include/linux/ke
#define TAINT_PROPRIETARY_MODULE 0
diff -Nur linux-4.1.10.orig/include/linux/kvm_host.h linux-4.1.10/include/linux/kvm_host.h
--- linux-4.1.10.orig/include/linux/kvm_host.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/kvm_host.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/kvm_host.h 2015-10-12 22:33:32.276676754 +0200
@@ -230,7 +230,7 @@
int fpu_active;
@@ -19185,7 +9928,7 @@ diff -Nur linux-4.1.10.orig/include/linux/kvm_host.h linux-4.1.10/include/linux/
return vcpu->arch.wqp;
diff -Nur linux-4.1.10.orig/include/linux/lglock.h linux-4.1.10/include/linux/lglock.h
--- linux-4.1.10.orig/include/linux/lglock.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/lglock.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/lglock.h 2015-10-12 22:33:32.276676754 +0200
@@ -34,22 +34,39 @@
#endif
@@ -19243,7 +9986,7 @@ diff -Nur linux-4.1.10.orig/include/linux/lglock.h linux-4.1.10/include/linux/lg
#define lglock spinlock
diff -Nur linux-4.1.10.orig/include/linux/list_bl.h linux-4.1.10/include/linux/list_bl.h
--- linux-4.1.10.orig/include/linux/list_bl.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/list_bl.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/list_bl.h 2015-10-12 22:33:32.276676754 +0200
@@ -2,6 +2,7 @@
#define _LINUX_LIST_BL_H
@@ -19306,7 +10049,7 @@ diff -Nur linux-4.1.10.orig/include/linux/list_bl.h linux-4.1.10/include/linux/l
static inline bool hlist_bl_is_locked(struct hlist_bl_head *b)
diff -Nur linux-4.1.10.orig/include/linux/locallock.h linux-4.1.10/include/linux/locallock.h
--- linux-4.1.10.orig/include/linux/locallock.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/locallock.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/locallock.h 2015-10-12 22:33:32.276676754 +0200
@@ -0,0 +1,270 @@
+#ifndef _LINUX_LOCALLOCK_H
+#define _LINUX_LOCALLOCK_H
@@ -19580,7 +10323,7 @@ diff -Nur linux-4.1.10.orig/include/linux/locallock.h linux-4.1.10/include/linux
+#endif
diff -Nur linux-4.1.10.orig/include/linux/mm_types.h linux-4.1.10/include/linux/mm_types.h
--- linux-4.1.10.orig/include/linux/mm_types.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/mm_types.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/mm_types.h 2015-10-12 22:33:32.276676754 +0200
@@ -11,6 +11,7 @@
#include <linux/completion.h>
#include <linux/cpumask.h>
@@ -19599,552 +10342,9 @@ diff -Nur linux-4.1.10.orig/include/linux/mm_types.h linux-4.1.10/include/linux/
#ifdef CONFIG_X86_INTEL_MPX
/* address of the bounds directory */
void __user *bd_addr;
-diff -Nur linux-4.1.10.orig/include/linux/mm_types.h.orig linux-4.1.10/include/linux/mm_types.h.orig
---- linux-4.1.10.orig/include/linux/mm_types.h.orig 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/mm_types.h.orig 2015-10-03 13:49:38.000000000 +0200
-@@ -0,0 +1,539 @@
-+#ifndef _LINUX_MM_TYPES_H
-+#define _LINUX_MM_TYPES_H
-+
-+#include <linux/auxvec.h>
-+#include <linux/types.h>
-+#include <linux/threads.h>
-+#include <linux/list.h>
-+#include <linux/spinlock.h>
-+#include <linux/rbtree.h>
-+#include <linux/rwsem.h>
-+#include <linux/completion.h>
-+#include <linux/cpumask.h>
-+#include <linux/uprobes.h>
-+#include <linux/page-flags-layout.h>
-+#include <asm/page.h>
-+#include <asm/mmu.h>
-+
-+#ifndef AT_VECTOR_SIZE_ARCH
-+#define AT_VECTOR_SIZE_ARCH 0
-+#endif
-+#define AT_VECTOR_SIZE (2*(AT_VECTOR_SIZE_ARCH + AT_VECTOR_SIZE_BASE + 1))
-+
-+struct address_space;
-+struct mem_cgroup;
-+
-+#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
-+#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \
-+ IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
-+#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8)
-+
-+typedef void compound_page_dtor(struct page *);
-+
-+/*
-+ * Each physical page in the system has a struct page associated with
-+ * it to keep track of whatever it is we are using the page for at the
-+ * moment. Note that we have no way to track which tasks are using
-+ * a page, though if it is a pagecache page, rmap structures can tell us
-+ * who is mapping it.
-+ *
-+ * The objects in struct page are organized in double word blocks in
-+ * order to allows us to use atomic double word operations on portions
-+ * of struct page. That is currently only used by slub but the arrangement
-+ * allows the use of atomic double word operations on the flags/mapping
-+ * and lru list pointers also.
-+ */
-+struct page {
-+ /* First double word block */
-+ unsigned long flags; /* Atomic flags, some possibly
-+ * updated asynchronously */
-+ union {
-+ struct address_space *mapping; /* If low bit clear, points to
-+ * inode address_space, or NULL.
-+ * If page mapped as anonymous
-+ * memory, low bit is set, and
-+ * it points to anon_vma object:
-+ * see PAGE_MAPPING_ANON below.
-+ */
-+ void *s_mem; /* slab first object */
-+ };
-+
-+ /* Second double word */
-+ struct {
-+ union {
-+ pgoff_t index; /* Our offset within mapping. */
-+ void *freelist; /* sl[aou]b first free object */
-+ };
-+
-+ union {
-+#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
-+ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
-+ /* Used for cmpxchg_double in slub */
-+ unsigned long counters;
-+#else
-+ /*
-+ * Keep _count separate from slub cmpxchg_double data.
-+ * As the rest of the double word is protected by
-+ * slab_lock but _count is not.
-+ */
-+ unsigned counters;
-+#endif
-+
-+ struct {
-+
-+ union {
-+ /*
-+ * Count of ptes mapped in
-+ * mms, to show when page is
-+ * mapped & limit reverse map
-+ * searches.
-+ *
-+ * Used also for tail pages
-+ * refcounting instead of
-+ * _count. Tail pages cannot
-+ * be mapped and keeping the
-+ * tail page _count zero at
-+ * all times guarantees
-+ * get_page_unless_zero() will
-+ * never succeed on tail
-+ * pages.
-+ */
-+ atomic_t _mapcount;
-+
-+ struct { /* SLUB */
-+ unsigned inuse:16;
-+ unsigned objects:15;
-+ unsigned frozen:1;
-+ };
-+ int units; /* SLOB */
-+ };
-+ atomic_t _count; /* Usage count, see below. */
-+ };
-+ unsigned int active; /* SLAB */
-+ };
-+ };
-+
-+ /* Third double word block */
-+ union {
-+ struct list_head lru; /* Pageout list, eg. active_list
-+ * protected by zone->lru_lock !
-+ * Can be used as a generic list
-+ * by the page owner.
-+ */
-+ struct { /* slub per cpu partial pages */
-+ struct page *next; /* Next partial slab */
-+#ifdef CONFIG_64BIT
-+ int pages; /* Nr of partial slabs left */
-+ int pobjects; /* Approximate # of objects */
-+#else
-+ short int pages;
-+ short int pobjects;
-+#endif
-+ };
-+
-+ struct slab *slab_page; /* slab fields */
-+ struct rcu_head rcu_head; /* Used by SLAB
-+ * when destroying via RCU
-+ */
-+ /* First tail page of compound page */
-+ struct {
-+ compound_page_dtor *compound_dtor;
-+ unsigned long compound_order;
-+ };
-+
-+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
-+ pgtable_t pmd_huge_pte; /* protected by page->ptl */
-+#endif
-+ };
-+
-+ /* Remainder is not double word aligned */
-+ union {
-+ unsigned long private; /* Mapping-private opaque data:
-+ * usually used for buffer_heads
-+ * if PagePrivate set; used for
-+ * swp_entry_t if PageSwapCache;
-+ * indicates order in the buddy
-+ * system if PG_buddy is set.
-+ */
-+#if USE_SPLIT_PTE_PTLOCKS
-+#if ALLOC_SPLIT_PTLOCKS
-+ spinlock_t *ptl;
-+#else
-+ spinlock_t ptl;
-+#endif
-+#endif
-+ struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */
-+ struct page *first_page; /* Compound tail pages */
-+ };
-+
-+#ifdef CONFIG_MEMCG
-+ struct mem_cgroup *mem_cgroup;
-+#endif
-+
-+ /*
-+ * On machines where all RAM is mapped into kernel address space,
-+ * we can simply calculate the virtual address. On machines with
-+ * highmem some memory is mapped into kernel virtual memory
-+ * dynamically, so we need a place to store that address.
-+ * Note that this field could be 16 bits on x86 ... ;)
-+ *
-+ * Architectures with slow multiplication can define
-+ * WANT_PAGE_VIRTUAL in asm/page.h
-+ */
-+#if defined(WANT_PAGE_VIRTUAL)
-+ void *virtual; /* Kernel virtual address (NULL if
-+ not kmapped, ie. highmem) */
-+#endif /* WANT_PAGE_VIRTUAL */
-+
-+#ifdef CONFIG_KMEMCHECK
-+ /*
-+ * kmemcheck wants to track the status of each byte in a page; this
-+ * is a pointer to such a status block. NULL if not tracked.
-+ */
-+ void *shadow;
-+#endif
-+
-+#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
-+ int _last_cpupid;
-+#endif
-+}
-+/*
-+ * The struct page can be forced to be double word aligned so that atomic ops
-+ * on double words work. The SLUB allocator can make use of such a feature.
-+ */
-+#ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE
-+ __aligned(2 * sizeof(unsigned long))
-+#endif
-+;
-+
-+struct page_frag {
-+ struct page *page;
-+#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
-+ __u32 offset;
-+ __u32 size;
-+#else
-+ __u16 offset;
-+ __u16 size;
-+#endif
-+};
-+
-+typedef unsigned long __nocast vm_flags_t;
-+
-+/*
-+ * A region containing a mapping of a non-memory backed file under NOMMU
-+ * conditions. These are held in a global tree and are pinned by the VMAs that
-+ * map parts of them.
-+ */
-+struct vm_region {
-+ struct rb_node vm_rb; /* link in global region tree */
-+ vm_flags_t vm_flags; /* VMA vm_flags */
-+ unsigned long vm_start; /* start address of region */
-+ unsigned long vm_end; /* region initialised to here */
-+ unsigned long vm_top; /* region allocated to here */
-+ unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */
-+ struct file *vm_file; /* the backing file or NULL */
-+
-+ int vm_usage; /* region usage count (access under nommu_region_sem) */
-+ bool vm_icache_flushed : 1; /* true if the icache has been flushed for
-+ * this region */
-+};
-+
-+/*
-+ * This struct defines a memory VMM memory area. There is one of these
-+ * per VM-area/task. A VM area is any part of the process virtual memory
-+ * space that has a special rule for the page-fault handlers (ie a shared
-+ * library, the executable area etc).
-+ */
-+struct vm_area_struct {
-+ /* The first cache line has the info for VMA tree walking. */
-+
-+ unsigned long vm_start; /* Our start address within vm_mm. */
-+ unsigned long vm_end; /* The first byte after our end address
-+ within vm_mm. */
-+
-+ /* linked list of VM areas per task, sorted by address */
-+ struct vm_area_struct *vm_next, *vm_prev;
-+
-+ struct rb_node vm_rb;
-+
-+ /*
-+ * Largest free memory gap in bytes to the left of this VMA.
-+ * Either between this VMA and vma->vm_prev, or between one of the
-+ * VMAs below us in the VMA rbtree and its ->vm_prev. This helps
-+ * get_unmapped_area find a free area of the right size.
-+ */
-+ unsigned long rb_subtree_gap;
-+
-+ /* Second cache line starts here. */
-+
-+ struct mm_struct *vm_mm; /* The address space we belong to. */
-+ pgprot_t vm_page_prot; /* Access permissions of this VMA. */
-+ unsigned long vm_flags; /* Flags, see mm.h. */
-+
-+ /*
-+ * For areas with an address space and backing store,
-+ * linkage into the address_space->i_mmap interval tree.
-+ */
-+ struct {
-+ struct rb_node rb;
-+ unsigned long rb_subtree_last;
-+ } shared;
-+
-+ /*
-+ * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
-+ * list, after a COW of one of the file pages. A MAP_SHARED vma
-+ * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack
-+ * or brk vma (with NULL file) can only be in an anon_vma list.
-+ */
-+ struct list_head anon_vma_chain; /* Serialized by mmap_sem &
-+ * page_table_lock */
-+ struct anon_vma *anon_vma; /* Serialized by page_table_lock */
-+
-+ /* Function pointers to deal with this struct. */
-+ const struct vm_operations_struct *vm_ops;
-+
-+ /* Information about our backing store: */
-+ unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE
-+ units, *not* PAGE_CACHE_SIZE */
-+ struct file * vm_file; /* File we map to (can be NULL). */
-+ void * vm_private_data; /* was vm_pte (shared mem) */
-+
-+#ifndef CONFIG_MMU
-+ struct vm_region *vm_region; /* NOMMU mapping region */
-+#endif
-+#ifdef CONFIG_NUMA
-+ struct mempolicy *vm_policy; /* NUMA policy for the VMA */
-+#endif
-+};
-+
-+struct core_thread {
-+ struct task_struct *task;
-+ struct core_thread *next;
-+};
-+
-+struct core_state {
-+ atomic_t nr_threads;
-+ struct core_thread dumper;
-+ struct completion startup;
-+};
-+
-+enum {
-+ MM_FILEPAGES,
-+ MM_ANONPAGES,
-+ MM_SWAPENTS,
-+ NR_MM_COUNTERS
-+};
-+
-+#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU)
-+#define SPLIT_RSS_COUNTING
-+/* per-thread cached information, */
-+struct task_rss_stat {
-+ int events; /* for synchronization threshold */
-+ int count[NR_MM_COUNTERS];
-+};
-+#endif /* USE_SPLIT_PTE_PTLOCKS */
-+
-+struct mm_rss_stat {
-+ atomic_long_t count[NR_MM_COUNTERS];
-+};
-+
-+struct kioctx_table;
-+struct mm_struct {
-+ struct vm_area_struct *mmap; /* list of VMAs */
-+ struct rb_root mm_rb;
-+ u32 vmacache_seqnum; /* per-thread vmacache */
-+#ifdef CONFIG_MMU
-+ unsigned long (*get_unmapped_area) (struct file *filp,
-+ unsigned long addr, unsigned long len,
-+ unsigned long pgoff, unsigned long flags);
-+#endif
-+ unsigned long mmap_base; /* base of mmap area */
-+ unsigned long mmap_legacy_base; /* base of mmap area in bottom-up allocations */
-+ unsigned long task_size; /* size of task vm space */
-+ unsigned long highest_vm_end; /* highest vma end address */
-+ pgd_t * pgd;
-+ atomic_t mm_users; /* How many users with user space? */
-+ atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */
-+ atomic_long_t nr_ptes; /* PTE page table pages */
-+#if CONFIG_PGTABLE_LEVELS > 2
-+ atomic_long_t nr_pmds; /* PMD page table pages */
-+#endif
-+ int map_count; /* number of VMAs */
-+
-+ spinlock_t page_table_lock; /* Protects page tables and some counters */
-+ struct rw_semaphore mmap_sem;
-+
-+ struct list_head mmlist; /* List of maybe swapped mm's. These are globally strung
-+ * together off init_mm.mmlist, and are protected
-+ * by mmlist_lock
-+ */
-+
-+
-+ unsigned long hiwater_rss; /* High-watermark of RSS usage */
-+ unsigned long hiwater_vm; /* High-water virtual memory usage */
-+
-+ unsigned long total_vm; /* Total pages mapped */
-+ unsigned long locked_vm; /* Pages that have PG_mlocked set */
-+ unsigned long pinned_vm; /* Refcount permanently increased */
-+ unsigned long shared_vm; /* Shared pages (files) */
-+ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE */
-+ unsigned long stack_vm; /* VM_GROWSUP/DOWN */
-+ unsigned long def_flags;
-+ unsigned long start_code, end_code, start_data, end_data;
-+ unsigned long start_brk, brk, start_stack;
-+ unsigned long arg_start, arg_end, env_start, env_end;
-+
-+ unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */
-+
-+ /*
-+ * Special counters, in some configurations protected by the
-+ * page_table_lock, in other configurations by being atomic.
-+ */
-+ struct mm_rss_stat rss_stat;
-+
-+ struct linux_binfmt *binfmt;
-+
-+ cpumask_var_t cpu_vm_mask_var;
-+
-+ /* Architecture-specific MM context */
-+ mm_context_t context;
-+
-+ unsigned long flags; /* Must use atomic bitops to access the bits */
-+
-+ struct core_state *core_state; /* coredumping support */
-+#ifdef CONFIG_AIO
-+ spinlock_t ioctx_lock;
-+ struct kioctx_table __rcu *ioctx_table;
-+#endif
-+#ifdef CONFIG_MEMCG
-+ /*
-+ * "owner" points to a task that is regarded as the canonical
-+ * user/owner of this mm. All of the following must be true in
-+ * order for it to be changed:
-+ *
-+ * current == mm->owner
-+ * current->mm != mm
-+ * new_owner->mm == mm
-+ * new_owner->alloc_lock is held
-+ */
-+ struct task_struct __rcu *owner;
-+#endif
-+
-+ /* store ref to file /proc/<pid>/exe symlink points to */
-+ struct file __rcu *exe_file;
-+#ifdef CONFIG_MMU_NOTIFIER
-+ struct mmu_notifier_mm *mmu_notifier_mm;
-+#endif
-+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
-+ pgtable_t pmd_huge_pte; /* protected by page_table_lock */
-+#endif
-+#ifdef CONFIG_CPUMASK_OFFSTACK
-+ struct cpumask cpumask_allocation;
-+#endif
-+#ifdef CONFIG_NUMA_BALANCING
-+ /*
-+ * numa_next_scan is the next time that the PTEs will be marked
-+ * pte_numa. NUMA hinting faults will gather statistics and migrate
-+ * pages to new nodes if necessary.
-+ */
-+ unsigned long numa_next_scan;
-+
-+ /* Restart point for scanning and setting pte_numa */
-+ unsigned long numa_scan_offset;
-+
-+ /* numa_scan_seq prevents two threads setting pte_numa */
-+ int numa_scan_seq;
-+#endif
-+#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION)
-+ /*
-+ * An operation with batched TLB flushing is going on. Anything that
-+ * can move process memory needs to flush the TLB when moving a
-+ * PROT_NONE or PROT_NUMA mapped page.
-+ */
-+ bool tlb_flush_pending;
-+#endif
-+ struct uprobes_state uprobes_state;
-+#ifdef CONFIG_X86_INTEL_MPX
-+ /* address of the bounds directory */
-+ void __user *bd_addr;
-+#endif
-+};
-+
-+static inline void mm_init_cpumask(struct mm_struct *mm)
-+{
-+#ifdef CONFIG_CPUMASK_OFFSTACK
-+ mm->cpu_vm_mask_var = &mm->cpumask_allocation;
-+#endif
-+ cpumask_clear(mm->cpu_vm_mask_var);
-+}
-+
-+/* Future-safe accessor for struct mm_struct's cpu_vm_mask. */
-+static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
-+{
-+ return mm->cpu_vm_mask_var;
-+}
-+
-+#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION)
-+/*
-+ * Memory barriers to keep this state in sync are graciously provided by
-+ * the page table locks, outside of which no page table modifications happen.
-+ * The barriers below prevent the compiler from re-ordering the instructions
-+ * around the memory barriers that are already present in the code.
-+ */
-+static inline bool mm_tlb_flush_pending(struct mm_struct *mm)
-+{
-+ barrier();
-+ return mm->tlb_flush_pending;
-+}
-+static inline void set_tlb_flush_pending(struct mm_struct *mm)
-+{
-+ mm->tlb_flush_pending = true;
-+
-+ /*
-+ * Guarantee that the tlb_flush_pending store does not leak into the
-+ * critical section updating the page tables
-+ */
-+ smp_mb__before_spinlock();
-+}
-+/* Clearing is done after a TLB flush, which also provides a barrier. */
-+static inline void clear_tlb_flush_pending(struct mm_struct *mm)
-+{
-+ barrier();
-+ mm->tlb_flush_pending = false;
-+}
-+#else
-+static inline bool mm_tlb_flush_pending(struct mm_struct *mm)
-+{
-+ return false;
-+}
-+static inline void set_tlb_flush_pending(struct mm_struct *mm)
-+{
-+}
-+static inline void clear_tlb_flush_pending(struct mm_struct *mm)
-+{
-+}
-+#endif
-+
-+struct vm_special_mapping
-+{
-+ const char *name;
-+ struct page **pages;
-+};
-+
-+enum tlb_flush_reason {
-+ TLB_FLUSH_ON_TASK_SWITCH,
-+ TLB_REMOTE_SHOOTDOWN,
-+ TLB_LOCAL_SHOOTDOWN,
-+ TLB_LOCAL_MM_SHOOTDOWN,
-+ NR_TLB_FLUSH_REASONS,
-+};
-+
-+ /*
-+ * A swap entry has to fit into a "unsigned long", as the entry is hidden
-+ * in the "index" field of the swapper address space.
-+ */
-+typedef struct {
-+ unsigned long val;
-+} swp_entry_t;
-+
-+#endif /* _LINUX_MM_TYPES_H */
diff -Nur linux-4.1.10.orig/include/linux/mutex.h linux-4.1.10/include/linux/mutex.h
--- linux-4.1.10.orig/include/linux/mutex.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/mutex.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/mutex.h 2015-10-12 22:33:32.276676754 +0200
@@ -19,6 +19,17 @@
#include <asm/processor.h>
#include <linux/osq_lock.h>
@@ -20188,7 +10388,7 @@ diff -Nur linux-4.1.10.orig/include/linux/mutex.h linux-4.1.10/include/linux/mut
#endif /* __LINUX_MUTEX_H */
diff -Nur linux-4.1.10.orig/include/linux/mutex_rt.h linux-4.1.10/include/linux/mutex_rt.h
--- linux-4.1.10.orig/include/linux/mutex_rt.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/mutex_rt.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/mutex_rt.h 2015-10-12 22:33:32.276676754 +0200
@@ -0,0 +1,84 @@
+#ifndef __LINUX_MUTEX_RT_H
+#define __LINUX_MUTEX_RT_H
@@ -20276,7 +10476,7 @@ diff -Nur linux-4.1.10.orig/include/linux/mutex_rt.h linux-4.1.10/include/linux/
+#endif
diff -Nur linux-4.1.10.orig/include/linux/netdevice.h linux-4.1.10/include/linux/netdevice.h
--- linux-4.1.10.orig/include/linux/netdevice.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/netdevice.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/netdevice.h 2015-10-12 22:33:32.276676754 +0200
@@ -2469,6 +2469,7 @@
unsigned int dropped;
struct sk_buff_head input_pkt_queue;
@@ -20287,7 +10487,7 @@ diff -Nur linux-4.1.10.orig/include/linux/netdevice.h linux-4.1.10/include/linux
diff -Nur linux-4.1.10.orig/include/linux/netfilter/x_tables.h linux-4.1.10/include/linux/netfilter/x_tables.h
--- linux-4.1.10.orig/include/linux/netfilter/x_tables.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/netfilter/x_tables.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/netfilter/x_tables.h 2015-10-12 22:33:32.276676754 +0200
@@ -3,6 +3,7 @@
@@ -20325,7 +10525,7 @@ diff -Nur linux-4.1.10.orig/include/linux/netfilter/x_tables.h linux-4.1.10/incl
/*
diff -Nur linux-4.1.10.orig/include/linux/notifier.h linux-4.1.10/include/linux/notifier.h
--- linux-4.1.10.orig/include/linux/notifier.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/notifier.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/notifier.h 2015-10-12 22:33:32.276676754 +0200
@@ -6,7 +6,7 @@
*
* Alan Cox <Alan.Cox@linux.org>
@@ -20407,7 +10607,7 @@ diff -Nur linux-4.1.10.orig/include/linux/notifier.h linux-4.1.10/include/linux/
/* netdevice notifiers are defined in include/linux/netdevice.h */
diff -Nur linux-4.1.10.orig/include/linux/percpu.h linux-4.1.10/include/linux/percpu.h
--- linux-4.1.10.orig/include/linux/percpu.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/percpu.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/percpu.h 2015-10-12 22:33:32.276676754 +0200
@@ -24,6 +24,35 @@
PERCPU_MODULE_RESERVE)
#endif
@@ -20446,7 +10646,7 @@ diff -Nur linux-4.1.10.orig/include/linux/percpu.h linux-4.1.10/include/linux/pe
diff -Nur linux-4.1.10.orig/include/linux/pid.h linux-4.1.10/include/linux/pid.h
--- linux-4.1.10.orig/include/linux/pid.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/pid.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/pid.h 2015-10-12 22:33:32.276676754 +0200
@@ -2,6 +2,7 @@
#define _LINUX_PID_H
@@ -20457,7 +10657,7 @@ diff -Nur linux-4.1.10.orig/include/linux/pid.h linux-4.1.10/include/linux/pid.h
{
diff -Nur linux-4.1.10.orig/include/linux/preempt.h linux-4.1.10/include/linux/preempt.h
--- linux-4.1.10.orig/include/linux/preempt.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/preempt.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/preempt.h 2015-10-12 22:33:32.276676754 +0200
@@ -33,6 +33,20 @@
#define preempt_count_inc() preempt_count_add(1)
#define preempt_count_dec() preempt_count_sub(1)
@@ -20563,7 +10763,7 @@ diff -Nur linux-4.1.10.orig/include/linux/preempt.h linux-4.1.10/include/linux/p
struct preempt_notifier;
diff -Nur linux-4.1.10.orig/include/linux/preempt_mask.h linux-4.1.10/include/linux/preempt_mask.h
--- linux-4.1.10.orig/include/linux/preempt_mask.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/preempt_mask.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/preempt_mask.h 2015-10-12 22:33:32.276676754 +0200
@@ -44,16 +44,26 @@
#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
#define NMI_OFFSET (1UL << NMI_SHIFT)
@@ -20603,7 +10803,7 @@ diff -Nur linux-4.1.10.orig/include/linux/preempt_mask.h linux-4.1.10/include/li
* Are we in NMI context?
diff -Nur linux-4.1.10.orig/include/linux/printk.h linux-4.1.10/include/linux/printk.h
--- linux-4.1.10.orig/include/linux/printk.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/printk.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/printk.h 2015-10-12 22:33:32.276676754 +0200
@@ -115,9 +115,11 @@
#ifdef CONFIG_EARLY_PRINTK
extern asmlinkage __printf(1, 2)
@@ -20618,7 +10818,7 @@ diff -Nur linux-4.1.10.orig/include/linux/printk.h linux-4.1.10/include/linux/pr
typedef int(*printk_func_t)(const char *fmt, va_list args);
diff -Nur linux-4.1.10.orig/include/linux/radix-tree.h linux-4.1.10/include/linux/radix-tree.h
--- linux-4.1.10.orig/include/linux/radix-tree.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/radix-tree.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/radix-tree.h 2015-10-12 22:33:32.280676490 +0200
@@ -277,8 +277,13 @@
unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
void ***results, unsigned long *indices,
@@ -20644,7 +10844,7 @@ diff -Nur linux-4.1.10.orig/include/linux/radix-tree.h linux-4.1.10/include/linu
/**
diff -Nur linux-4.1.10.orig/include/linux/random.h linux-4.1.10/include/linux/random.h
--- linux-4.1.10.orig/include/linux/random.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/random.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/random.h 2015-10-12 22:33:32.280676490 +0200
@@ -11,7 +11,7 @@
extern void add_device_randomness(const void *, unsigned int);
extern void add_input_randomness(unsigned int type, unsigned int code,
@@ -20656,7 +10856,7 @@ diff -Nur linux-4.1.10.orig/include/linux/random.h linux-4.1.10/include/linux/ra
extern void get_random_bytes_arch(void *buf, int nbytes);
diff -Nur linux-4.1.10.orig/include/linux/rcupdate.h linux-4.1.10/include/linux/rcupdate.h
--- linux-4.1.10.orig/include/linux/rcupdate.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/rcupdate.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/rcupdate.h 2015-10-12 22:33:32.280676490 +0200
@@ -167,6 +167,9 @@
#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
@@ -20743,7 +10943,7 @@ diff -Nur linux-4.1.10.orig/include/linux/rcupdate.h linux-4.1.10/include/linux/
diff -Nur linux-4.1.10.orig/include/linux/rcutree.h linux-4.1.10/include/linux/rcutree.h
--- linux-4.1.10.orig/include/linux/rcutree.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/rcutree.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/rcutree.h 2015-10-12 22:33:32.280676490 +0200
@@ -46,7 +46,11 @@
rcu_note_context_switch();
}
@@ -20798,7 +10998,7 @@ diff -Nur linux-4.1.10.orig/include/linux/rcutree.h linux-4.1.10/include/linux/r
#endif /* __LINUX_RCUTREE_H */
diff -Nur linux-4.1.10.orig/include/linux/rtmutex.h linux-4.1.10/include/linux/rtmutex.h
--- linux-4.1.10.orig/include/linux/rtmutex.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/rtmutex.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/rtmutex.h 2015-10-12 22:33:32.280676490 +0200
@@ -14,10 +14,14 @@
#include <linux/linkage.h>
@@ -20874,7 +11074,7 @@ diff -Nur linux-4.1.10.orig/include/linux/rtmutex.h linux-4.1.10/include/linux/r
diff -Nur linux-4.1.10.orig/include/linux/rwlock_rt.h linux-4.1.10/include/linux/rwlock_rt.h
--- linux-4.1.10.orig/include/linux/rwlock_rt.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/rwlock_rt.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/rwlock_rt.h 2015-10-12 22:33:32.280676490 +0200
@@ -0,0 +1,99 @@
+#ifndef __LINUX_RWLOCK_RT_H
+#define __LINUX_RWLOCK_RT_H
@@ -20977,7 +11177,7 @@ diff -Nur linux-4.1.10.orig/include/linux/rwlock_rt.h linux-4.1.10/include/linux
+#endif
diff -Nur linux-4.1.10.orig/include/linux/rwlock_types.h linux-4.1.10/include/linux/rwlock_types.h
--- linux-4.1.10.orig/include/linux/rwlock_types.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/rwlock_types.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/rwlock_types.h 2015-10-12 22:33:32.280676490 +0200
@@ -1,6 +1,10 @@
#ifndef __LINUX_RWLOCK_TYPES_H
#define __LINUX_RWLOCK_TYPES_H
@@ -21000,7 +11200,7 @@ diff -Nur linux-4.1.10.orig/include/linux/rwlock_types.h linux-4.1.10/include/li
#endif /* __LINUX_RWLOCK_TYPES_H */
diff -Nur linux-4.1.10.orig/include/linux/rwlock_types_rt.h linux-4.1.10/include/linux/rwlock_types_rt.h
--- linux-4.1.10.orig/include/linux/rwlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/rwlock_types_rt.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/rwlock_types_rt.h 2015-10-12 22:33:32.280676490 +0200
@@ -0,0 +1,33 @@
+#ifndef __LINUX_RWLOCK_TYPES_RT_H
+#define __LINUX_RWLOCK_TYPES_RT_H
@@ -21037,7 +11237,7 @@ diff -Nur linux-4.1.10.orig/include/linux/rwlock_types_rt.h linux-4.1.10/include
+#endif
diff -Nur linux-4.1.10.orig/include/linux/rwsem.h linux-4.1.10/include/linux/rwsem.h
--- linux-4.1.10.orig/include/linux/rwsem.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/rwsem.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/rwsem.h 2015-10-12 22:33:32.280676490 +0200
@@ -18,6 +18,10 @@
#include <linux/osq_lock.h>
#endif
@@ -21058,7 +11258,7 @@ diff -Nur linux-4.1.10.orig/include/linux/rwsem.h linux-4.1.10/include/linux/rws
#endif /* _LINUX_RWSEM_H */
diff -Nur linux-4.1.10.orig/include/linux/rwsem_rt.h linux-4.1.10/include/linux/rwsem_rt.h
--- linux-4.1.10.orig/include/linux/rwsem_rt.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/rwsem_rt.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/rwsem_rt.h 2015-10-12 22:33:32.280676490 +0200
@@ -0,0 +1,140 @@
+#ifndef _LINUX_RWSEM_RT_H
+#define _LINUX_RWSEM_RT_H
@@ -21202,7 +11402,7 @@ diff -Nur linux-4.1.10.orig/include/linux/rwsem_rt.h linux-4.1.10/include/linux/
+#endif
diff -Nur linux-4.1.10.orig/include/linux/sched.h linux-4.1.10/include/linux/sched.h
--- linux-4.1.10.orig/include/linux/sched.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/sched.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/sched.h 2015-10-12 22:33:32.280676490 +0200
@@ -26,6 +26,7 @@
#include <linux/nodemask.h>
#include <linux/mm_types.h>
@@ -21628,7 +11828,7 @@ diff -Nur linux-4.1.10.orig/include/linux/sched.h linux-4.1.10/include/linux/sch
diff -Nur linux-4.1.10.orig/include/linux/seqlock.h linux-4.1.10/include/linux/seqlock.h
--- linux-4.1.10.orig/include/linux/seqlock.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/seqlock.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/seqlock.h 2015-10-12 22:33:32.280676490 +0200
@@ -219,20 +219,30 @@
return __read_seqcount_retry(s, start);
}
@@ -21760,7 +11960,7 @@ diff -Nur linux-4.1.10.orig/include/linux/seqlock.h linux-4.1.10/include/linux/s
diff -Nur linux-4.1.10.orig/include/linux/signal.h linux-4.1.10/include/linux/signal.h
--- linux-4.1.10.orig/include/linux/signal.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/signal.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/signal.h 2015-10-12 22:33:32.280676490 +0200
@@ -218,6 +218,7 @@
}
@@ -21771,7 +11971,7 @@ diff -Nur linux-4.1.10.orig/include/linux/signal.h linux-4.1.10/include/linux/si
static inline int valid_signal(unsigned long sig)
diff -Nur linux-4.1.10.orig/include/linux/skbuff.h linux-4.1.10/include/linux/skbuff.h
--- linux-4.1.10.orig/include/linux/skbuff.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/skbuff.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/skbuff.h 2015-10-12 22:33:32.280676490 +0200
@@ -187,6 +187,7 @@
__u32 qlen;
@@ -21795,7 +11995,7 @@ diff -Nur linux-4.1.10.orig/include/linux/skbuff.h linux-4.1.10/include/linux/sk
{
diff -Nur linux-4.1.10.orig/include/linux/smp.h linux-4.1.10/include/linux/smp.h
--- linux-4.1.10.orig/include/linux/smp.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/smp.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/smp.h 2015-10-12 22:33:32.280676490 +0200
@@ -185,6 +185,9 @@
#define get_cpu() ({ preempt_disable(); smp_processor_id(); })
#define put_cpu() preempt_enable()
@@ -21808,7 +12008,7 @@ diff -Nur linux-4.1.10.orig/include/linux/smp.h linux-4.1.10/include/linux/smp.h
* boot command line:
diff -Nur linux-4.1.10.orig/include/linux/spinlock_api_smp.h linux-4.1.10/include/linux/spinlock_api_smp.h
--- linux-4.1.10.orig/include/linux/spinlock_api_smp.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/spinlock_api_smp.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/spinlock_api_smp.h 2015-10-12 22:33:32.280676490 +0200
@@ -189,6 +189,8 @@
return 0;
}
@@ -21821,7 +12021,7 @@ diff -Nur linux-4.1.10.orig/include/linux/spinlock_api_smp.h linux-4.1.10/includ
#endif /* __LINUX_SPINLOCK_API_SMP_H */
diff -Nur linux-4.1.10.orig/include/linux/spinlock.h linux-4.1.10/include/linux/spinlock.h
--- linux-4.1.10.orig/include/linux/spinlock.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/spinlock.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/spinlock.h 2015-10-12 22:33:32.280676490 +0200
@@ -281,7 +281,11 @@
#define raw_spin_can_lock(lock) (!raw_spin_is_locked(lock))
@@ -21855,7 +12055,7 @@ diff -Nur linux-4.1.10.orig/include/linux/spinlock.h linux-4.1.10/include/linux/
#endif /* __LINUX_SPINLOCK_H */
diff -Nur linux-4.1.10.orig/include/linux/spinlock_rt.h linux-4.1.10/include/linux/spinlock_rt.h
--- linux-4.1.10.orig/include/linux/spinlock_rt.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/spinlock_rt.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/spinlock_rt.h 2015-10-12 22:33:32.280676490 +0200
@@ -0,0 +1,174 @@
+#ifndef __LINUX_SPINLOCK_RT_H
+#define __LINUX_SPINLOCK_RT_H
@@ -22033,7 +12233,7 @@ diff -Nur linux-4.1.10.orig/include/linux/spinlock_rt.h linux-4.1.10/include/lin
+#endif
diff -Nur linux-4.1.10.orig/include/linux/spinlock_types.h linux-4.1.10/include/linux/spinlock_types.h
--- linux-4.1.10.orig/include/linux/spinlock_types.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/spinlock_types.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/spinlock_types.h 2015-10-12 22:33:32.280676490 +0200
@@ -9,80 +9,15 @@
* Released under the General Public License (GPL).
*/
@@ -22124,7 +12324,7 @@ diff -Nur linux-4.1.10.orig/include/linux/spinlock_types.h linux-4.1.10/include/
#endif /* __LINUX_SPINLOCK_TYPES_H */
diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_nort.h linux-4.1.10/include/linux/spinlock_types_nort.h
--- linux-4.1.10.orig/include/linux/spinlock_types_nort.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/spinlock_types_nort.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/spinlock_types_nort.h 2015-10-12 22:33:32.280676490 +0200
@@ -0,0 +1,33 @@
+#ifndef __LINUX_SPINLOCK_TYPES_NORT_H
+#define __LINUX_SPINLOCK_TYPES_NORT_H
@@ -22161,7 +12361,7 @@ diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_nort.h linux-4.1.10/inc
+#endif
diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_raw.h linux-4.1.10/include/linux/spinlock_types_raw.h
--- linux-4.1.10.orig/include/linux/spinlock_types_raw.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/spinlock_types_raw.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/spinlock_types_raw.h 2015-10-12 22:33:32.284676225 +0200
@@ -0,0 +1,56 @@
+#ifndef __LINUX_SPINLOCK_TYPES_RAW_H
+#define __LINUX_SPINLOCK_TYPES_RAW_H
@@ -22221,7 +12421,7 @@ diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_raw.h linux-4.1.10/incl
+#endif
diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_rt.h linux-4.1.10/include/linux/spinlock_types_rt.h
--- linux-4.1.10.orig/include/linux/spinlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/spinlock_types_rt.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/spinlock_types_rt.h 2015-10-12 22:33:32.284676225 +0200
@@ -0,0 +1,51 @@
+#ifndef __LINUX_SPINLOCK_TYPES_RT_H
+#define __LINUX_SPINLOCK_TYPES_RT_H
@@ -22276,7 +12476,7 @@ diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_rt.h linux-4.1.10/inclu
+#endif
diff -Nur linux-4.1.10.orig/include/linux/srcu.h linux-4.1.10/include/linux/srcu.h
--- linux-4.1.10.orig/include/linux/srcu.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/srcu.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/srcu.h 2015-10-12 22:33:32.284676225 +0200
@@ -84,10 +84,10 @@
void process_srcu(struct work_struct *work);
@@ -22301,7 +12501,7 @@ diff -Nur linux-4.1.10.orig/include/linux/srcu.h linux-4.1.10/include/linux/srcu
diff -Nur linux-4.1.10.orig/include/linux/swap.h linux-4.1.10/include/linux/swap.h
--- linux-4.1.10.orig/include/linux/swap.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/swap.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/swap.h 2015-10-12 22:33:32.284676225 +0200
@@ -11,6 +11,7 @@
#include <linux/fs.h>
#include <linux/atomic.h>
@@ -22330,7 +12530,7 @@ diff -Nur linux-4.1.10.orig/include/linux/swap.h linux-4.1.10/include/linux/swap
extern void lru_cache_add_file(struct page *page);
diff -Nur linux-4.1.10.orig/include/linux/thread_info.h linux-4.1.10/include/linux/thread_info.h
--- linux-4.1.10.orig/include/linux/thread_info.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/thread_info.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/thread_info.h 2015-10-12 22:33:32.284676225 +0200
@@ -102,7 +102,17 @@
#define test_thread_flag(flag) \
test_ti_thread_flag(current_thread_info(), flag)
@@ -22352,7 +12552,7 @@ diff -Nur linux-4.1.10.orig/include/linux/thread_info.h linux-4.1.10/include/lin
/*
diff -Nur linux-4.1.10.orig/include/linux/timer.h linux-4.1.10/include/linux/timer.h
--- linux-4.1.10.orig/include/linux/timer.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/timer.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/timer.h 2015-10-12 22:33:32.284676225 +0200
@@ -241,7 +241,7 @@
extern int try_to_del_timer_sync(struct timer_list *timer);
@@ -22364,7 +12564,7 @@ diff -Nur linux-4.1.10.orig/include/linux/timer.h linux-4.1.10/include/linux/tim
# define del_timer_sync(t) del_timer(t)
diff -Nur linux-4.1.10.orig/include/linux/uaccess.h linux-4.1.10/include/linux/uaccess.h
--- linux-4.1.10.orig/include/linux/uaccess.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/uaccess.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/uaccess.h 2015-10-12 22:33:32.284676225 +0200
@@ -1,21 +1,31 @@
#ifndef __LINUX_UACCESS_H__
#define __LINUX_UACCESS_H__
@@ -22445,7 +12645,7 @@ diff -Nur linux-4.1.10.orig/include/linux/uaccess.h linux-4.1.10/include/linux/u
static inline unsigned long __copy_from_user_inatomic_nocache(void *to,
diff -Nur linux-4.1.10.orig/include/linux/uprobes.h linux-4.1.10/include/linux/uprobes.h
--- linux-4.1.10.orig/include/linux/uprobes.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/uprobes.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/uprobes.h 2015-10-12 22:33:32.284676225 +0200
@@ -27,6 +27,7 @@
#include <linux/errno.h>
#include <linux/rbtree.h>
@@ -22456,7 +12656,7 @@ diff -Nur linux-4.1.10.orig/include/linux/uprobes.h linux-4.1.10/include/linux/u
struct mm_struct;
diff -Nur linux-4.1.10.orig/include/linux/vmstat.h linux-4.1.10/include/linux/vmstat.h
--- linux-4.1.10.orig/include/linux/vmstat.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/vmstat.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/vmstat.h 2015-10-12 22:33:32.284676225 +0200
@@ -33,7 +33,9 @@
*/
static inline void __count_vm_event(enum vm_event_item item)
@@ -22479,7 +12679,7 @@ diff -Nur linux-4.1.10.orig/include/linux/vmstat.h linux-4.1.10/include/linux/vm
static inline void count_vm_events(enum vm_event_item item, long delta)
diff -Nur linux-4.1.10.orig/include/linux/wait.h linux-4.1.10/include/linux/wait.h
--- linux-4.1.10.orig/include/linux/wait.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/linux/wait.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/wait.h 2015-10-12 22:33:32.284676225 +0200
@@ -8,6 +8,7 @@
#include <linux/spinlock.h>
#include <asm/current.h>
@@ -22490,7 +12690,7 @@ diff -Nur linux-4.1.10.orig/include/linux/wait.h linux-4.1.10/include/linux/wait
typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key);
diff -Nur linux-4.1.10.orig/include/linux/wait-simple.h linux-4.1.10/include/linux/wait-simple.h
--- linux-4.1.10.orig/include/linux/wait-simple.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/wait-simple.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/wait-simple.h 2015-10-12 22:33:32.284676225 +0200
@@ -0,0 +1,207 @@
+#ifndef _LINUX_WAIT_SIMPLE_H
+#define _LINUX_WAIT_SIMPLE_H
@@ -22701,7 +12901,7 @@ diff -Nur linux-4.1.10.orig/include/linux/wait-simple.h linux-4.1.10/include/lin
+#endif
diff -Nur linux-4.1.10.orig/include/linux/work-simple.h linux-4.1.10/include/linux/work-simple.h
--- linux-4.1.10.orig/include/linux/work-simple.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/linux/work-simple.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/linux/work-simple.h 2015-10-12 22:33:32.284676225 +0200
@@ -0,0 +1,24 @@
+#ifndef _LINUX_SWORK_H
+#define _LINUX_SWORK_H
@@ -22729,7 +12929,7 @@ diff -Nur linux-4.1.10.orig/include/linux/work-simple.h linux-4.1.10/include/lin
+#endif /* _LINUX_SWORK_H */
diff -Nur linux-4.1.10.orig/include/net/dst.h linux-4.1.10/include/net/dst.h
--- linux-4.1.10.orig/include/net/dst.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/net/dst.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/net/dst.h 2015-10-12 22:33:32.284676225 +0200
@@ -403,7 +403,7 @@
static inline int dst_neigh_output(struct dst_entry *dst, struct neighbour *n,
struct sk_buff *skb)
@@ -22741,7 +12941,7 @@ diff -Nur linux-4.1.10.orig/include/net/dst.h linux-4.1.10/include/net/dst.h
unsigned long now = jiffies;
diff -Nur linux-4.1.10.orig/include/net/neighbour.h linux-4.1.10/include/net/neighbour.h
--- linux-4.1.10.orig/include/net/neighbour.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/net/neighbour.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/net/neighbour.h 2015-10-12 22:33:32.284676225 +0200
@@ -445,7 +445,7 @@
}
#endif
@@ -22762,7 +12962,7 @@ diff -Nur linux-4.1.10.orig/include/net/neighbour.h linux-4.1.10/include/net/nei
unsigned int seq;
diff -Nur linux-4.1.10.orig/include/net/netns/ipv4.h linux-4.1.10/include/net/netns/ipv4.h
--- linux-4.1.10.orig/include/net/netns/ipv4.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/include/net/netns/ipv4.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/net/netns/ipv4.h 2015-10-12 22:33:32.284676225 +0200
@@ -69,6 +69,7 @@
int sysctl_icmp_echo_ignore_all;
@@ -22773,7 +12973,7 @@ diff -Nur linux-4.1.10.orig/include/net/netns/ipv4.h linux-4.1.10/include/net/ne
int sysctl_icmp_ratemask;
diff -Nur linux-4.1.10.orig/include/trace/events/hist.h linux-4.1.10/include/trace/events/hist.h
--- linux-4.1.10.orig/include/trace/events/hist.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/trace/events/hist.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/trace/events/hist.h 2015-10-12 22:33:32.284676225 +0200
@@ -0,0 +1,72 @@
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM hist
@@ -22849,7 +13049,7 @@ diff -Nur linux-4.1.10.orig/include/trace/events/hist.h linux-4.1.10/include/tra
+#include <trace/define_trace.h>
diff -Nur linux-4.1.10.orig/include/trace/events/latency_hist.h linux-4.1.10/include/trace/events/latency_hist.h
--- linux-4.1.10.orig/include/trace/events/latency_hist.h 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/include/trace/events/latency_hist.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/include/trace/events/latency_hist.h 2015-10-12 22:33:32.284676225 +0200
@@ -0,0 +1,29 @@
+#ifndef _LATENCY_HIST_H
+#define _LATENCY_HIST_H
@@ -22882,7 +13082,7 @@ diff -Nur linux-4.1.10.orig/include/trace/events/latency_hist.h linux-4.1.10/inc
+#endif /* _LATENCY_HIST_H */
diff -Nur linux-4.1.10.orig/init/Kconfig linux-4.1.10/init/Kconfig
--- linux-4.1.10.orig/init/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/init/Kconfig 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/init/Kconfig 2015-10-12 22:33:32.284676225 +0200
@@ -637,7 +637,7 @@
config RCU_FAST_NO_HZ
@@ -22936,7 +13136,7 @@ diff -Nur linux-4.1.10.orig/init/Kconfig linux-4.1.10/init/Kconfig
Per cpu partial caches accellerate objects allocation and freeing
diff -Nur linux-4.1.10.orig/init/main.c linux-4.1.10/init/main.c
--- linux-4.1.10.orig/init/main.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/init/main.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/init/main.c 2015-10-12 22:33:32.284676225 +0200
@@ -525,6 +525,7 @@
setup_command_line(command_line);
setup_nr_cpu_ids();
@@ -22947,7 +13147,7 @@ diff -Nur linux-4.1.10.orig/init/main.c linux-4.1.10/init/main.c
build_all_zonelists(NULL, NULL);
diff -Nur linux-4.1.10.orig/init/Makefile linux-4.1.10/init/Makefile
--- linux-4.1.10.orig/init/Makefile 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/init/Makefile 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/init/Makefile 2015-10-12 22:33:32.284676225 +0200
@@ -33,4 +33,4 @@
include/generated/compile.h: FORCE
@$($(quiet)chk_compile.h)
@@ -22956,7 +13156,7 @@ diff -Nur linux-4.1.10.orig/init/Makefile linux-4.1.10/init/Makefile
+ "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CONFIG_PREEMPT_RT_FULL)" "$(CC) $(KBUILD_CFLAGS)"
diff -Nur linux-4.1.10.orig/ipc/mqueue.c linux-4.1.10/ipc/mqueue.c
--- linux-4.1.10.orig/ipc/mqueue.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/ipc/mqueue.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/ipc/mqueue.c 2015-10-12 22:33:32.284676225 +0200
@@ -47,8 +47,7 @@
#define RECV 1
@@ -23099,7 +13299,7 @@ diff -Nur linux-4.1.10.orig/ipc/mqueue.c linux-4.1.10/ipc/mqueue.c
if (ret == 0) {
diff -Nur linux-4.1.10.orig/ipc/msg.c linux-4.1.10/ipc/msg.c
--- linux-4.1.10.orig/ipc/msg.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/ipc/msg.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/ipc/msg.c 2015-10-12 22:33:32.288675961 +0200
@@ -188,6 +188,12 @@
struct msg_receiver *msr, *t;
@@ -23151,7 +13351,7 @@ diff -Nur linux-4.1.10.orig/ipc/msg.c linux-4.1.10/ipc/msg.c
diff -Nur linux-4.1.10.orig/ipc/sem.c linux-4.1.10/ipc/sem.c
--- linux-4.1.10.orig/ipc/sem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/ipc/sem.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/ipc/sem.c 2015-10-12 22:33:32.288675961 +0200
@@ -690,6 +690,13 @@
static void wake_up_sem_queue_prepare(struct list_head *pt,
struct sem_queue *q, int error)
@@ -23192,7 +13392,7 @@ diff -Nur linux-4.1.10.orig/ipc/sem.c linux-4.1.10/ipc/sem.c
static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
diff -Nur linux-4.1.10.orig/kernel/cgroup.c linux-4.1.10/kernel/cgroup.c
--- linux-4.1.10.orig/kernel/cgroup.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/cgroup.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/cgroup.c 2015-10-12 22:33:32.288675961 +0200
@@ -4422,10 +4422,10 @@
queue_work(cgroup_destroy_wq, &css->destroy_work);
}
@@ -23225,5615 +13425,9 @@ diff -Nur linux-4.1.10.orig/kernel/cgroup.c linux-4.1.10/kernel/cgroup.c
/*
* Used to destroy pidlists and separate to serve as flush domain.
-diff -Nur linux-4.1.10.orig/kernel/cgroup.c.orig linux-4.1.10/kernel/cgroup.c.orig
---- linux-4.1.10.orig/kernel/cgroup.c.orig 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/kernel/cgroup.c.orig 2015-10-03 13:49:38.000000000 +0200
-@@ -0,0 +1,5602 @@
-+/*
-+ * Generic process-grouping system.
-+ *
-+ * Based originally on the cpuset system, extracted by Paul Menage
-+ * Copyright (C) 2006 Google, Inc
-+ *
-+ * Notifications support
-+ * Copyright (C) 2009 Nokia Corporation
-+ * Author: Kirill A. Shutemov
-+ *
-+ * Copyright notices from the original cpuset code:
-+ * --------------------------------------------------
-+ * Copyright (C) 2003 BULL SA.
-+ * Copyright (C) 2004-2006 Silicon Graphics, Inc.
-+ *
-+ * Portions derived from Patrick Mochel's sysfs code.
-+ * sysfs is Copyright (c) 2001-3 Patrick Mochel
-+ *
-+ * 2003-10-10 Written by Simon Derr.
-+ * 2003-10-22 Updates by Stephen Hemminger.
-+ * 2004 May-July Rework by Paul Jackson.
-+ * ---------------------------------------------------
-+ *
-+ * This file is subject to the terms and conditions of the GNU General Public
-+ * License. See the file COPYING in the main directory of the Linux
-+ * distribution for more details.
-+ */
-+
-+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-+
-+#include <linux/cgroup.h>
-+#include <linux/cred.h>
-+#include <linux/ctype.h>
-+#include <linux/errno.h>
-+#include <linux/init_task.h>
-+#include <linux/kernel.h>
-+#include <linux/list.h>
-+#include <linux/magic.h>
-+#include <linux/mm.h>
-+#include <linux/mutex.h>
-+#include <linux/mount.h>
-+#include <linux/pagemap.h>
-+#include <linux/proc_fs.h>
-+#include <linux/rcupdate.h>
-+#include <linux/sched.h>
-+#include <linux/slab.h>
-+#include <linux/spinlock.h>
-+#include <linux/rwsem.h>
-+#include <linux/string.h>
-+#include <linux/sort.h>
-+#include <linux/kmod.h>
-+#include <linux/delayacct.h>
-+#include <linux/cgroupstats.h>
-+#include <linux/hashtable.h>
-+#include <linux/pid_namespace.h>
-+#include <linux/idr.h>
-+#include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
-+#include <linux/kthread.h>
-+#include <linux/delay.h>
-+
-+#include <linux/atomic.h>
-+
-+/*
-+ * pidlists linger the following amount before being destroyed. The goal
-+ * is avoiding frequent destruction in the middle of consecutive read calls
-+ * Expiring in the middle is a performance problem not a correctness one.
-+ * 1 sec should be enough.
-+ */
-+#define CGROUP_PIDLIST_DESTROY_DELAY HZ
-+
-+#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \
-+ MAX_CFTYPE_NAME + 2)
-+
-+/*
-+ * cgroup_mutex is the master lock. Any modification to cgroup or its
-+ * hierarchy must be performed while holding it.
-+ *
-+ * css_set_rwsem protects task->cgroups pointer, the list of css_set
-+ * objects, and the chain of tasks off each css_set.
-+ *
-+ * These locks are exported if CONFIG_PROVE_RCU so that accessors in
-+ * cgroup.h can use them for lockdep annotations.
-+ */
-+#ifdef CONFIG_PROVE_RCU
-+DEFINE_MUTEX(cgroup_mutex);
-+DECLARE_RWSEM(css_set_rwsem);
-+EXPORT_SYMBOL_GPL(cgroup_mutex);
-+EXPORT_SYMBOL_GPL(css_set_rwsem);
-+#else
-+static DEFINE_MUTEX(cgroup_mutex);
-+static DECLARE_RWSEM(css_set_rwsem);
-+#endif
-+
-+/*
-+ * Protects cgroup_idr and css_idr so that IDs can be released without
-+ * grabbing cgroup_mutex.
-+ */
-+static DEFINE_SPINLOCK(cgroup_idr_lock);
-+
-+/*
-+ * Protects cgroup_subsys->release_agent_path. Modifying it also requires
-+ * cgroup_mutex. Reading requires either cgroup_mutex or this spinlock.
-+ */
-+static DEFINE_SPINLOCK(release_agent_path_lock);
-+
-+#define cgroup_assert_mutex_or_rcu_locked() \
-+ rcu_lockdep_assert(rcu_read_lock_held() || \
-+ lockdep_is_held(&cgroup_mutex), \
-+ "cgroup_mutex or RCU read lock required");
-+
-+/*
-+ * cgroup destruction makes heavy use of work items and there can be a lot
-+ * of concurrent destructions. Use a separate workqueue so that cgroup
-+ * destruction work items don't end up filling up max_active of system_wq
-+ * which may lead to deadlock.
-+ */
-+static struct workqueue_struct *cgroup_destroy_wq;
-+
-+/*
-+ * pidlist destructions need to be flushed on cgroup destruction. Use a
-+ * separate workqueue as flush domain.
-+ */
-+static struct workqueue_struct *cgroup_pidlist_destroy_wq;
-+
-+/* generate an array of cgroup subsystem pointers */
-+#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
-+static struct cgroup_subsys *cgroup_subsys[] = {
-+#include <linux/cgroup_subsys.h>
-+};
-+#undef SUBSYS
-+
-+/* array of cgroup subsystem names */
-+#define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
-+static const char *cgroup_subsys_name[] = {
-+#include <linux/cgroup_subsys.h>
-+};
-+#undef SUBSYS
-+
-+/*
-+ * The default hierarchy, reserved for the subsystems that are otherwise
-+ * unattached - it never has more than a single cgroup, and all tasks are
-+ * part of that cgroup.
-+ */
-+struct cgroup_root cgrp_dfl_root;
-+
-+/*
-+ * The default hierarchy always exists but is hidden until mounted for the
-+ * first time. This is for backward compatibility.
-+ */
-+static bool cgrp_dfl_root_visible;
-+
-+/*
-+ * Set by the boot param of the same name and makes subsystems with NULL
-+ * ->dfl_files to use ->legacy_files on the default hierarchy.
-+ */
-+static bool cgroup_legacy_files_on_dfl;
-+
-+/* some controllers are not supported in the default hierarchy */
-+static unsigned int cgrp_dfl_root_inhibit_ss_mask;
-+
-+/* The list of hierarchy roots */
-+
-+static LIST_HEAD(cgroup_roots);
-+static int cgroup_root_count;
-+
-+/* hierarchy ID allocation and mapping, protected by cgroup_mutex */
-+static DEFINE_IDR(cgroup_hierarchy_idr);
-+
-+/*
-+ * Assign a monotonically increasing serial number to csses. It guarantees
-+ * cgroups with bigger numbers are newer than those with smaller numbers.
-+ * Also, as csses are always appended to the parent's ->children list, it
-+ * guarantees that sibling csses are always sorted in the ascending serial
-+ * number order on the list. Protected by cgroup_mutex.
-+ */
-+static u64 css_serial_nr_next = 1;
-+
-+/* This flag indicates whether tasks in the fork and exit paths should
-+ * check for fork/exit handlers to call. This avoids us having to do
-+ * extra work in the fork/exit path if none of the subsystems need to
-+ * be called.
-+ */
-+static int need_forkexit_callback __read_mostly;
-+
-+static struct cftype cgroup_dfl_base_files[];
-+static struct cftype cgroup_legacy_base_files[];
-+
-+static int rebind_subsystems(struct cgroup_root *dst_root,
-+ unsigned int ss_mask);
-+static int cgroup_destroy_locked(struct cgroup *cgrp);
-+static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
-+ bool visible);
-+static void css_release(struct percpu_ref *ref);
-+static void kill_css(struct cgroup_subsys_state *css);
-+static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
-+ bool is_add);
-+
-+/* IDR wrappers which synchronize using cgroup_idr_lock */
-+static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end,
-+ gfp_t gfp_mask)
-+{
-+ int ret;
-+
-+ idr_preload(gfp_mask);
-+ spin_lock_bh(&cgroup_idr_lock);
-+ ret = idr_alloc(idr, ptr, start, end, gfp_mask);
-+ spin_unlock_bh(&cgroup_idr_lock);
-+ idr_preload_end();
-+ return ret;
-+}
-+
-+static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id)
-+{
-+ void *ret;
-+
-+ spin_lock_bh(&cgroup_idr_lock);
-+ ret = idr_replace(idr, ptr, id);
-+ spin_unlock_bh(&cgroup_idr_lock);
-+ return ret;
-+}
-+
-+static void cgroup_idr_remove(struct idr *idr, int id)
-+{
-+ spin_lock_bh(&cgroup_idr_lock);
-+ idr_remove(idr, id);
-+ spin_unlock_bh(&cgroup_idr_lock);
-+}
-+
-+static struct cgroup *cgroup_parent(struct cgroup *cgrp)
-+{
-+ struct cgroup_subsys_state *parent_css = cgrp->self.parent;
-+
-+ if (parent_css)
-+ return container_of(parent_css, struct cgroup, self);
-+ return NULL;
-+}
-+
-+/**
-+ * cgroup_css - obtain a cgroup's css for the specified subsystem
-+ * @cgrp: the cgroup of interest
-+ * @ss: the subsystem of interest (%NULL returns @cgrp->self)
-+ *
-+ * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This
-+ * function must be called either under cgroup_mutex or rcu_read_lock() and
-+ * the caller is responsible for pinning the returned css if it wants to
-+ * keep accessing it outside the said locks. This function may return
-+ * %NULL if @cgrp doesn't have @subsys_id enabled.
-+ */
-+static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
-+ struct cgroup_subsys *ss)
-+{
-+ if (ss)
-+ return rcu_dereference_check(cgrp->subsys[ss->id],
-+ lockdep_is_held(&cgroup_mutex));
-+ else
-+ return &cgrp->self;
-+}
-+
-+/**
-+ * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
-+ * @cgrp: the cgroup of interest
-+ * @ss: the subsystem of interest (%NULL returns @cgrp->self)
-+ *
-+ * Similar to cgroup_css() but returns the effctive css, which is defined
-+ * as the matching css of the nearest ancestor including self which has @ss
-+ * enabled. If @ss is associated with the hierarchy @cgrp is on, this
-+ * function is guaranteed to return non-NULL css.
-+ */
-+static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
-+ struct cgroup_subsys *ss)
-+{
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ if (!ss)
-+ return &cgrp->self;
-+
-+ if (!(cgrp->root->subsys_mask & (1 << ss->id)))
-+ return NULL;
-+
-+ /*
-+ * This function is used while updating css associations and thus
-+ * can't test the csses directly. Use ->child_subsys_mask.
-+ */
-+ while (cgroup_parent(cgrp) &&
-+ !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ss->id)))
-+ cgrp = cgroup_parent(cgrp);
-+
-+ return cgroup_css(cgrp, ss);
-+}
-+
-+/**
-+ * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
-+ * @cgrp: the cgroup of interest
-+ * @ss: the subsystem of interest
-+ *
-+ * Find and get the effective css of @cgrp for @ss. The effective css is
-+ * defined as the matching css of the nearest ancestor including self which
-+ * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on,
-+ * the root css is returned, so this function always returns a valid css.
-+ * The returned css must be put using css_put().
-+ */
-+struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp,
-+ struct cgroup_subsys *ss)
-+{
-+ struct cgroup_subsys_state *css;
-+
-+ rcu_read_lock();
-+
-+ do {
-+ css = cgroup_css(cgrp, ss);
-+
-+ if (css && css_tryget_online(css))
-+ goto out_unlock;
-+ cgrp = cgroup_parent(cgrp);
-+ } while (cgrp);
-+
-+ css = init_css_set.subsys[ss->id];
-+ css_get(css);
-+out_unlock:
-+ rcu_read_unlock();
-+ return css;
-+}
-+
-+/* convenient tests for these bits */
-+static inline bool cgroup_is_dead(const struct cgroup *cgrp)
-+{
-+ return !(cgrp->self.flags & CSS_ONLINE);
-+}
-+
-+struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
-+{
-+ struct cgroup *cgrp = of->kn->parent->priv;
-+ struct cftype *cft = of_cft(of);
-+
-+ /*
-+ * This is open and unprotected implementation of cgroup_css().
-+ * seq_css() is only called from a kernfs file operation which has
-+ * an active reference on the file. Because all the subsystem
-+ * files are drained before a css is disassociated with a cgroup,
-+ * the matching css from the cgroup's subsys table is guaranteed to
-+ * be and stay valid until the enclosing operation is complete.
-+ */
-+ if (cft->ss)
-+ return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
-+ else
-+ return &cgrp->self;
-+}
-+EXPORT_SYMBOL_GPL(of_css);
-+
-+/**
-+ * cgroup_is_descendant - test ancestry
-+ * @cgrp: the cgroup to be tested
-+ * @ancestor: possible ancestor of @cgrp
-+ *
-+ * Test whether @cgrp is a descendant of @ancestor. It also returns %true
-+ * if @cgrp == @ancestor. This function is safe to call as long as @cgrp
-+ * and @ancestor are accessible.
-+ */
-+bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor)
-+{
-+ while (cgrp) {
-+ if (cgrp == ancestor)
-+ return true;
-+ cgrp = cgroup_parent(cgrp);
-+ }
-+ return false;
-+}
-+
-+static int notify_on_release(const struct cgroup *cgrp)
-+{
-+ return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
-+}
-+
-+/**
-+ * for_each_css - iterate all css's of a cgroup
-+ * @css: the iteration cursor
-+ * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
-+ * @cgrp: the target cgroup to iterate css's of
-+ *
-+ * Should be called under cgroup_[tree_]mutex.
-+ */
-+#define for_each_css(css, ssid, cgrp) \
-+ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \
-+ if (!((css) = rcu_dereference_check( \
-+ (cgrp)->subsys[(ssid)], \
-+ lockdep_is_held(&cgroup_mutex)))) { } \
-+ else
-+
-+/**
-+ * for_each_e_css - iterate all effective css's of a cgroup
-+ * @css: the iteration cursor
-+ * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
-+ * @cgrp: the target cgroup to iterate css's of
-+ *
-+ * Should be called under cgroup_[tree_]mutex.
-+ */
-+#define for_each_e_css(css, ssid, cgrp) \
-+ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \
-+ if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
-+ ; \
-+ else
-+
-+/**
-+ * for_each_subsys - iterate all enabled cgroup subsystems
-+ * @ss: the iteration cursor
-+ * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
-+ */
-+#define for_each_subsys(ss, ssid) \
-+ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT && \
-+ (((ss) = cgroup_subsys[ssid]) || true); (ssid)++)
-+
-+/* iterate across the hierarchies */
-+#define for_each_root(root) \
-+ list_for_each_entry((root), &cgroup_roots, root_list)
-+
-+/* iterate over child cgrps, lock should be held throughout iteration */
-+#define cgroup_for_each_live_child(child, cgrp) \
-+ list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
-+ if (({ lockdep_assert_held(&cgroup_mutex); \
-+ cgroup_is_dead(child); })) \
-+ ; \
-+ else
-+
-+static void cgroup_release_agent(struct work_struct *work);
-+static void check_for_release(struct cgroup *cgrp);
-+
-+/*
-+ * A cgroup can be associated with multiple css_sets as different tasks may
-+ * belong to different cgroups on different hierarchies. In the other
-+ * direction, a css_set is naturally associated with multiple cgroups.
-+ * This M:N relationship is represented by the following link structure
-+ * which exists for each association and allows traversing the associations
-+ * from both sides.
-+ */
-+struct cgrp_cset_link {
-+ /* the cgroup and css_set this link associates */
-+ struct cgroup *cgrp;
-+ struct css_set *cset;
-+
-+ /* list of cgrp_cset_links anchored at cgrp->cset_links */
-+ struct list_head cset_link;
-+
-+ /* list of cgrp_cset_links anchored at css_set->cgrp_links */
-+ struct list_head cgrp_link;
-+};
-+
-+/*
-+ * The default css_set - used by init and its children prior to any
-+ * hierarchies being mounted. It contains a pointer to the root state
-+ * for each subsystem. Also used to anchor the list of css_sets. Not
-+ * reference-counted, to improve performance when child cgroups
-+ * haven't been created.
-+ */
-+struct css_set init_css_set = {
-+ .refcount = ATOMIC_INIT(1),
-+ .cgrp_links = LIST_HEAD_INIT(init_css_set.cgrp_links),
-+ .tasks = LIST_HEAD_INIT(init_css_set.tasks),
-+ .mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks),
-+ .mg_preload_node = LIST_HEAD_INIT(init_css_set.mg_preload_node),
-+ .mg_node = LIST_HEAD_INIT(init_css_set.mg_node),
-+};
-+
-+static int css_set_count = 1; /* 1 for init_css_set */
-+
-+/**
-+ * cgroup_update_populated - updated populated count of a cgroup
-+ * @cgrp: the target cgroup
-+ * @populated: inc or dec populated count
-+ *
-+ * @cgrp is either getting the first task (css_set) or losing the last.
-+ * Update @cgrp->populated_cnt accordingly. The count is propagated
-+ * towards root so that a given cgroup's populated_cnt is zero iff the
-+ * cgroup and all its descendants are empty.
-+ *
-+ * @cgrp's interface file "cgroup.populated" is zero if
-+ * @cgrp->populated_cnt is zero and 1 otherwise. When @cgrp->populated_cnt
-+ * changes from or to zero, userland is notified that the content of the
-+ * interface file has changed. This can be used to detect when @cgrp and
-+ * its descendants become populated or empty.
-+ */
-+static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
-+{
-+ lockdep_assert_held(&css_set_rwsem);
-+
-+ do {
-+ bool trigger;
-+
-+ if (populated)
-+ trigger = !cgrp->populated_cnt++;
-+ else
-+ trigger = !--cgrp->populated_cnt;
-+
-+ if (!trigger)
-+ break;
-+
-+ if (cgrp->populated_kn)
-+ kernfs_notify(cgrp->populated_kn);
-+ cgrp = cgroup_parent(cgrp);
-+ } while (cgrp);
-+}
-+
-+/*
-+ * hash table for cgroup groups. This improves the performance to find
-+ * an existing css_set. This hash doesn't (currently) take into
-+ * account cgroups in empty hierarchies.
-+ */
-+#define CSS_SET_HASH_BITS 7
-+static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
-+
-+static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
-+{
-+ unsigned long key = 0UL;
-+ struct cgroup_subsys *ss;
-+ int i;
-+
-+ for_each_subsys(ss, i)
-+ key += (unsigned long)css[i];
-+ key = (key >> 16) ^ key;
-+
-+ return key;
-+}
-+
-+static void put_css_set_locked(struct css_set *cset)
-+{
-+ struct cgrp_cset_link *link, *tmp_link;
-+ struct cgroup_subsys *ss;
-+ int ssid;
-+
-+ lockdep_assert_held(&css_set_rwsem);
-+
-+ if (!atomic_dec_and_test(&cset->refcount))
-+ return;
-+
-+ /* This css_set is dead. unlink it and release cgroup refcounts */
-+ for_each_subsys(ss, ssid)
-+ list_del(&cset->e_cset_node[ssid]);
-+ hash_del(&cset->hlist);
-+ css_set_count--;
-+
-+ list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
-+ struct cgroup *cgrp = link->cgrp;
-+
-+ list_del(&link->cset_link);
-+ list_del(&link->cgrp_link);
-+
-+ /* @cgrp can't go away while we're holding css_set_rwsem */
-+ if (list_empty(&cgrp->cset_links)) {
-+ cgroup_update_populated(cgrp, false);
-+ check_for_release(cgrp);
-+ }
-+
-+ kfree(link);
-+ }
-+
-+ kfree_rcu(cset, rcu_head);
-+}
-+
-+static void put_css_set(struct css_set *cset)
-+{
-+ /*
-+ * Ensure that the refcount doesn't hit zero while any readers
-+ * can see it. Similar to atomic_dec_and_lock(), but for an
-+ * rwlock
-+ */
-+ if (atomic_add_unless(&cset->refcount, -1, 1))
-+ return;
-+
-+ down_write(&css_set_rwsem);
-+ put_css_set_locked(cset);
-+ up_write(&css_set_rwsem);
-+}
-+
-+/*
-+ * refcounted get/put for css_set objects
-+ */
-+static inline void get_css_set(struct css_set *cset)
-+{
-+ atomic_inc(&cset->refcount);
-+}
-+
-+/**
-+ * compare_css_sets - helper function for find_existing_css_set().
-+ * @cset: candidate css_set being tested
-+ * @old_cset: existing css_set for a task
-+ * @new_cgrp: cgroup that's being entered by the task
-+ * @template: desired set of css pointers in css_set (pre-calculated)
-+ *
-+ * Returns true if "cset" matches "old_cset" except for the hierarchy
-+ * which "new_cgrp" belongs to, for which it should match "new_cgrp".
-+ */
-+static bool compare_css_sets(struct css_set *cset,
-+ struct css_set *old_cset,
-+ struct cgroup *new_cgrp,
-+ struct cgroup_subsys_state *template[])
-+{
-+ struct list_head *l1, *l2;
-+
-+ /*
-+ * On the default hierarchy, there can be csets which are
-+ * associated with the same set of cgroups but different csses.
-+ * Let's first ensure that csses match.
-+ */
-+ if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
-+ return false;
-+
-+ /*
-+ * Compare cgroup pointers in order to distinguish between
-+ * different cgroups in hierarchies. As different cgroups may
-+ * share the same effective css, this comparison is always
-+ * necessary.
-+ */
-+ l1 = &cset->cgrp_links;
-+ l2 = &old_cset->cgrp_links;
-+ while (1) {
-+ struct cgrp_cset_link *link1, *link2;
-+ struct cgroup *cgrp1, *cgrp2;
-+
-+ l1 = l1->next;
-+ l2 = l2->next;
-+ /* See if we reached the end - both lists are equal length. */
-+ if (l1 == &cset->cgrp_links) {
-+ BUG_ON(l2 != &old_cset->cgrp_links);
-+ break;
-+ } else {
-+ BUG_ON(l2 == &old_cset->cgrp_links);
-+ }
-+ /* Locate the cgroups associated with these links. */
-+ link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
-+ link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
-+ cgrp1 = link1->cgrp;
-+ cgrp2 = link2->cgrp;
-+ /* Hierarchies should be linked in the same order. */
-+ BUG_ON(cgrp1->root != cgrp2->root);
-+
-+ /*
-+ * If this hierarchy is the hierarchy of the cgroup
-+ * that's changing, then we need to check that this
-+ * css_set points to the new cgroup; if it's any other
-+ * hierarchy, then this css_set should point to the
-+ * same cgroup as the old css_set.
-+ */
-+ if (cgrp1->root == new_cgrp->root) {
-+ if (cgrp1 != new_cgrp)
-+ return false;
-+ } else {
-+ if (cgrp1 != cgrp2)
-+ return false;
-+ }
-+ }
-+ return true;
-+}
-+
-+/**
-+ * find_existing_css_set - init css array and find the matching css_set
-+ * @old_cset: the css_set that we're using before the cgroup transition
-+ * @cgrp: the cgroup that we're moving into
-+ * @template: out param for the new set of csses, should be clear on entry
-+ */
-+static struct css_set *find_existing_css_set(struct css_set *old_cset,
-+ struct cgroup *cgrp,
-+ struct cgroup_subsys_state *template[])
-+{
-+ struct cgroup_root *root = cgrp->root;
-+ struct cgroup_subsys *ss;
-+ struct css_set *cset;
-+ unsigned long key;
-+ int i;
-+
-+ /*
-+ * Build the set of subsystem state objects that we want to see in the
-+ * new css_set. while subsystems can change globally, the entries here
-+ * won't change, so no need for locking.
-+ */
-+ for_each_subsys(ss, i) {
-+ if (root->subsys_mask & (1UL << i)) {
-+ /*
-+ * @ss is in this hierarchy, so we want the
-+ * effective css from @cgrp.
-+ */
-+ template[i] = cgroup_e_css(cgrp, ss);
-+ } else {
-+ /*
-+ * @ss is not in this hierarchy, so we don't want
-+ * to change the css.
-+ */
-+ template[i] = old_cset->subsys[i];
-+ }
-+ }
-+
-+ key = css_set_hash(template);
-+ hash_for_each_possible(css_set_table, cset, hlist, key) {
-+ if (!compare_css_sets(cset, old_cset, cgrp, template))
-+ continue;
-+
-+ /* This css_set matches what we need */
-+ return cset;
-+ }
-+
-+ /* No existing cgroup group matched */
-+ return NULL;
-+}
-+
-+static void free_cgrp_cset_links(struct list_head *links_to_free)
-+{
-+ struct cgrp_cset_link *link, *tmp_link;
-+
-+ list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
-+ list_del(&link->cset_link);
-+ kfree(link);
-+ }
-+}
-+
-+/**
-+ * allocate_cgrp_cset_links - allocate cgrp_cset_links
-+ * @count: the number of links to allocate
-+ * @tmp_links: list_head the allocated links are put on
-+ *
-+ * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
-+ * through ->cset_link. Returns 0 on success or -errno.
-+ */
-+static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
-+{
-+ struct cgrp_cset_link *link;
-+ int i;
-+
-+ INIT_LIST_HEAD(tmp_links);
-+
-+ for (i = 0; i < count; i++) {
-+ link = kzalloc(sizeof(*link), GFP_KERNEL);
-+ if (!link) {
-+ free_cgrp_cset_links(tmp_links);
-+ return -ENOMEM;
-+ }
-+ list_add(&link->cset_link, tmp_links);
-+ }
-+ return 0;
-+}
-+
-+/**
-+ * link_css_set - a helper function to link a css_set to a cgroup
-+ * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
-+ * @cset: the css_set to be linked
-+ * @cgrp: the destination cgroup
-+ */
-+static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
-+ struct cgroup *cgrp)
-+{
-+ struct cgrp_cset_link *link;
-+
-+ BUG_ON(list_empty(tmp_links));
-+
-+ if (cgroup_on_dfl(cgrp))
-+ cset->dfl_cgrp = cgrp;
-+
-+ link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
-+ link->cset = cset;
-+ link->cgrp = cgrp;
-+
-+ if (list_empty(&cgrp->cset_links))
-+ cgroup_update_populated(cgrp, true);
-+ list_move(&link->cset_link, &cgrp->cset_links);
-+
-+ /*
-+ * Always add links to the tail of the list so that the list
-+ * is sorted by order of hierarchy creation
-+ */
-+ list_add_tail(&link->cgrp_link, &cset->cgrp_links);
-+}
-+
-+/**
-+ * find_css_set - return a new css_set with one cgroup updated
-+ * @old_cset: the baseline css_set
-+ * @cgrp: the cgroup to be updated
-+ *
-+ * Return a new css_set that's equivalent to @old_cset, but with @cgrp
-+ * substituted into the appropriate hierarchy.
-+ */
-+static struct css_set *find_css_set(struct css_set *old_cset,
-+ struct cgroup *cgrp)
-+{
-+ struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
-+ struct css_set *cset;
-+ struct list_head tmp_links;
-+ struct cgrp_cset_link *link;
-+ struct cgroup_subsys *ss;
-+ unsigned long key;
-+ int ssid;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ /* First see if we already have a cgroup group that matches
-+ * the desired set */
-+ down_read(&css_set_rwsem);
-+ cset = find_existing_css_set(old_cset, cgrp, template);
-+ if (cset)
-+ get_css_set(cset);
-+ up_read(&css_set_rwsem);
-+
-+ if (cset)
-+ return cset;
-+
-+ cset = kzalloc(sizeof(*cset), GFP_KERNEL);
-+ if (!cset)
-+ return NULL;
-+
-+ /* Allocate all the cgrp_cset_link objects that we'll need */
-+ if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
-+ kfree(cset);
-+ return NULL;
-+ }
-+
-+ atomic_set(&cset->refcount, 1);
-+ INIT_LIST_HEAD(&cset->cgrp_links);
-+ INIT_LIST_HEAD(&cset->tasks);
-+ INIT_LIST_HEAD(&cset->mg_tasks);
-+ INIT_LIST_HEAD(&cset->mg_preload_node);
-+ INIT_LIST_HEAD(&cset->mg_node);
-+ INIT_HLIST_NODE(&cset->hlist);
-+
-+ /* Copy the set of subsystem state objects generated in
-+ * find_existing_css_set() */
-+ memcpy(cset->subsys, template, sizeof(cset->subsys));
-+
-+ down_write(&css_set_rwsem);
-+ /* Add reference counts and links from the new css_set. */
-+ list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
-+ struct cgroup *c = link->cgrp;
-+
-+ if (c->root == cgrp->root)
-+ c = cgrp;
-+ link_css_set(&tmp_links, cset, c);
-+ }
-+
-+ BUG_ON(!list_empty(&tmp_links));
-+
-+ css_set_count++;
-+
-+ /* Add @cset to the hash table */
-+ key = css_set_hash(cset->subsys);
-+ hash_add(css_set_table, &cset->hlist, key);
-+
-+ for_each_subsys(ss, ssid)
-+ list_add_tail(&cset->e_cset_node[ssid],
-+ &cset->subsys[ssid]->cgroup->e_csets[ssid]);
-+
-+ up_write(&css_set_rwsem);
-+
-+ return cset;
-+}
-+
-+static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
-+{
-+ struct cgroup *root_cgrp = kf_root->kn->priv;
-+
-+ return root_cgrp->root;
-+}
-+
-+static int cgroup_init_root_id(struct cgroup_root *root)
-+{
-+ int id;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
-+ if (id < 0)
-+ return id;
-+
-+ root->hierarchy_id = id;
-+ return 0;
-+}
-+
-+static void cgroup_exit_root_id(struct cgroup_root *root)
-+{
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ if (root->hierarchy_id) {
-+ idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
-+ root->hierarchy_id = 0;
-+ }
-+}
-+
-+static void cgroup_free_root(struct cgroup_root *root)
-+{
-+ if (root) {
-+ /* hierarhcy ID shoulid already have been released */
-+ WARN_ON_ONCE(root->hierarchy_id);
-+
-+ idr_destroy(&root->cgroup_idr);
-+ kfree(root);
-+ }
-+}
-+
-+static void cgroup_destroy_root(struct cgroup_root *root)
-+{
-+ struct cgroup *cgrp = &root->cgrp;
-+ struct cgrp_cset_link *link, *tmp_link;
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ BUG_ON(atomic_read(&root->nr_cgrps));
-+ BUG_ON(!list_empty(&cgrp->self.children));
-+
-+ /* Rebind all subsystems back to the default hierarchy */
-+ rebind_subsystems(&cgrp_dfl_root, root->subsys_mask);
-+
-+ /*
-+ * Release all the links from cset_links to this hierarchy's
-+ * root cgroup
-+ */
-+ down_write(&css_set_rwsem);
-+
-+ list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
-+ list_del(&link->cset_link);
-+ list_del(&link->cgrp_link);
-+ kfree(link);
-+ }
-+ up_write(&css_set_rwsem);
-+
-+ if (!list_empty(&root->root_list)) {
-+ list_del(&root->root_list);
-+ cgroup_root_count--;
-+ }
-+
-+ cgroup_exit_root_id(root);
-+
-+ mutex_unlock(&cgroup_mutex);
-+
-+ kernfs_destroy_root(root->kf_root);
-+ cgroup_free_root(root);
-+}
-+
-+/* look up cgroup associated with given css_set on the specified hierarchy */
-+static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
-+ struct cgroup_root *root)
-+{
-+ struct cgroup *res = NULL;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+ lockdep_assert_held(&css_set_rwsem);
-+
-+ if (cset == &init_css_set) {
-+ res = &root->cgrp;
-+ } else {
-+ struct cgrp_cset_link *link;
-+
-+ list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
-+ struct cgroup *c = link->cgrp;
-+
-+ if (c->root == root) {
-+ res = c;
-+ break;
-+ }
-+ }
-+ }
-+
-+ BUG_ON(!res);
-+ return res;
-+}
-+
-+/*
-+ * Return the cgroup for "task" from the given hierarchy. Must be
-+ * called with cgroup_mutex and css_set_rwsem held.
-+ */
-+static struct cgroup *task_cgroup_from_root(struct task_struct *task,
-+ struct cgroup_root *root)
-+{
-+ /*
-+ * No need to lock the task - since we hold cgroup_mutex the
-+ * task can't change groups, so the only thing that can happen
-+ * is that it exits and its css is set back to init_css_set.
-+ */
-+ return cset_cgroup_from_root(task_css_set(task), root);
-+}
-+
-+/*
-+ * A task must hold cgroup_mutex to modify cgroups.
-+ *
-+ * Any task can increment and decrement the count field without lock.
-+ * So in general, code holding cgroup_mutex can't rely on the count
-+ * field not changing. However, if the count goes to zero, then only
-+ * cgroup_attach_task() can increment it again. Because a count of zero
-+ * means that no tasks are currently attached, therefore there is no
-+ * way a task attached to that cgroup can fork (the other way to
-+ * increment the count). So code holding cgroup_mutex can safely
-+ * assume that if the count is zero, it will stay zero. Similarly, if
-+ * a task holds cgroup_mutex on a cgroup with zero count, it
-+ * knows that the cgroup won't be removed, as cgroup_rmdir()
-+ * needs that mutex.
-+ *
-+ * A cgroup can only be deleted if both its 'count' of using tasks
-+ * is zero, and its list of 'children' cgroups is empty. Since all
-+ * tasks in the system use _some_ cgroup, and since there is always at
-+ * least one task in the system (init, pid == 1), therefore, root cgroup
-+ * always has either children cgroups and/or using tasks. So we don't
-+ * need a special hack to ensure that root cgroup cannot be deleted.
-+ *
-+ * P.S. One more locking exception. RCU is used to guard the
-+ * update of a tasks cgroup pointer by cgroup_attach_task()
-+ */
-+
-+static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask);
-+static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
-+static const struct file_operations proc_cgroupstats_operations;
-+
-+static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
-+ char *buf)
-+{
-+ if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
-+ !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
-+ snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
-+ cft->ss->name, cft->name);
-+ else
-+ strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
-+ return buf;
-+}
-+
-+/**
-+ * cgroup_file_mode - deduce file mode of a control file
-+ * @cft: the control file in question
-+ *
-+ * returns cft->mode if ->mode is not 0
-+ * returns S_IRUGO|S_IWUSR if it has both a read and a write handler
-+ * returns S_IRUGO if it has only a read handler
-+ * returns S_IWUSR if it has only a write hander
-+ */
-+static umode_t cgroup_file_mode(const struct cftype *cft)
-+{
-+ umode_t mode = 0;
-+
-+ if (cft->mode)
-+ return cft->mode;
-+
-+ if (cft->read_u64 || cft->read_s64 || cft->seq_show)
-+ mode |= S_IRUGO;
-+
-+ if (cft->write_u64 || cft->write_s64 || cft->write)
-+ mode |= S_IWUSR;
-+
-+ return mode;
-+}
-+
-+static void cgroup_get(struct cgroup *cgrp)
-+{
-+ WARN_ON_ONCE(cgroup_is_dead(cgrp));
-+ css_get(&cgrp->self);
-+}
-+
-+static bool cgroup_tryget(struct cgroup *cgrp)
-+{
-+ return css_tryget(&cgrp->self);
-+}
-+
-+static void cgroup_put(struct cgroup *cgrp)
-+{
-+ css_put(&cgrp->self);
-+}
-+
-+/**
-+ * cgroup_calc_child_subsys_mask - calculate child_subsys_mask
-+ * @cgrp: the target cgroup
-+ * @subtree_control: the new subtree_control mask to consider
-+ *
-+ * On the default hierarchy, a subsystem may request other subsystems to be
-+ * enabled together through its ->depends_on mask. In such cases, more
-+ * subsystems than specified in "cgroup.subtree_control" may be enabled.
-+ *
-+ * This function calculates which subsystems need to be enabled if
-+ * @subtree_control is to be applied to @cgrp. The returned mask is always
-+ * a superset of @subtree_control and follows the usual hierarchy rules.
-+ */
-+static unsigned int cgroup_calc_child_subsys_mask(struct cgroup *cgrp,
-+ unsigned int subtree_control)
-+{
-+ struct cgroup *parent = cgroup_parent(cgrp);
-+ unsigned int cur_ss_mask = subtree_control;
-+ struct cgroup_subsys *ss;
-+ int ssid;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ if (!cgroup_on_dfl(cgrp))
-+ return cur_ss_mask;
-+
-+ while (true) {
-+ unsigned int new_ss_mask = cur_ss_mask;
-+
-+ for_each_subsys(ss, ssid)
-+ if (cur_ss_mask & (1 << ssid))
-+ new_ss_mask |= ss->depends_on;
-+
-+ /*
-+ * Mask out subsystems which aren't available. This can
-+ * happen only if some depended-upon subsystems were bound
-+ * to non-default hierarchies.
-+ */
-+ if (parent)
-+ new_ss_mask &= parent->child_subsys_mask;
-+ else
-+ new_ss_mask &= cgrp->root->subsys_mask;
-+
-+ if (new_ss_mask == cur_ss_mask)
-+ break;
-+ cur_ss_mask = new_ss_mask;
-+ }
-+
-+ return cur_ss_mask;
-+}
-+
-+/**
-+ * cgroup_refresh_child_subsys_mask - update child_subsys_mask
-+ * @cgrp: the target cgroup
-+ *
-+ * Update @cgrp->child_subsys_mask according to the current
-+ * @cgrp->subtree_control using cgroup_calc_child_subsys_mask().
-+ */
-+static void cgroup_refresh_child_subsys_mask(struct cgroup *cgrp)
-+{
-+ cgrp->child_subsys_mask =
-+ cgroup_calc_child_subsys_mask(cgrp, cgrp->subtree_control);
-+}
-+
-+/**
-+ * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
-+ * @kn: the kernfs_node being serviced
-+ *
-+ * This helper undoes cgroup_kn_lock_live() and should be invoked before
-+ * the method finishes if locking succeeded. Note that once this function
-+ * returns the cgroup returned by cgroup_kn_lock_live() may become
-+ * inaccessible any time. If the caller intends to continue to access the
-+ * cgroup, it should pin it before invoking this function.
-+ */
-+static void cgroup_kn_unlock(struct kernfs_node *kn)
-+{
-+ struct cgroup *cgrp;
-+
-+ if (kernfs_type(kn) == KERNFS_DIR)
-+ cgrp = kn->priv;
-+ else
-+ cgrp = kn->parent->priv;
-+
-+ mutex_unlock(&cgroup_mutex);
-+
-+ kernfs_unbreak_active_protection(kn);
-+ cgroup_put(cgrp);
-+}
-+
-+/**
-+ * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
-+ * @kn: the kernfs_node being serviced
-+ *
-+ * This helper is to be used by a cgroup kernfs method currently servicing
-+ * @kn. It breaks the active protection, performs cgroup locking and
-+ * verifies that the associated cgroup is alive. Returns the cgroup if
-+ * alive; otherwise, %NULL. A successful return should be undone by a
-+ * matching cgroup_kn_unlock() invocation.
-+ *
-+ * Any cgroup kernfs method implementation which requires locking the
-+ * associated cgroup should use this helper. It avoids nesting cgroup
-+ * locking under kernfs active protection and allows all kernfs operations
-+ * including self-removal.
-+ */
-+static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn)
-+{
-+ struct cgroup *cgrp;
-+
-+ if (kernfs_type(kn) == KERNFS_DIR)
-+ cgrp = kn->priv;
-+ else
-+ cgrp = kn->parent->priv;
-+
-+ /*
-+ * We're gonna grab cgroup_mutex which nests outside kernfs
-+ * active_ref. cgroup liveliness check alone provides enough
-+ * protection against removal. Ensure @cgrp stays accessible and
-+ * break the active_ref protection.
-+ */
-+ if (!cgroup_tryget(cgrp))
-+ return NULL;
-+ kernfs_break_active_protection(kn);
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ if (!cgroup_is_dead(cgrp))
-+ return cgrp;
-+
-+ cgroup_kn_unlock(kn);
-+ return NULL;
-+}
-+
-+static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
-+{
-+ char name[CGROUP_FILE_NAME_MAX];
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+ kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
-+}
-+
-+/**
-+ * cgroup_clear_dir - remove subsys files in a cgroup directory
-+ * @cgrp: target cgroup
-+ * @subsys_mask: mask of the subsystem ids whose files should be removed
-+ */
-+static void cgroup_clear_dir(struct cgroup *cgrp, unsigned int subsys_mask)
-+{
-+ struct cgroup_subsys *ss;
-+ int i;
-+
-+ for_each_subsys(ss, i) {
-+ struct cftype *cfts;
-+
-+ if (!(subsys_mask & (1 << i)))
-+ continue;
-+ list_for_each_entry(cfts, &ss->cfts, node)
-+ cgroup_addrm_files(cgrp, cfts, false);
-+ }
-+}
-+
-+static int rebind_subsystems(struct cgroup_root *dst_root, unsigned int ss_mask)
-+{
-+ struct cgroup_subsys *ss;
-+ unsigned int tmp_ss_mask;
-+ int ssid, i, ret;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ for_each_subsys(ss, ssid) {
-+ if (!(ss_mask & (1 << ssid)))
-+ continue;
-+
-+ /* if @ss has non-root csses attached to it, can't move */
-+ if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)))
-+ return -EBUSY;
-+
-+ /* can't move between two non-dummy roots either */
-+ if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
-+ return -EBUSY;
-+ }
-+
-+ /* skip creating root files on dfl_root for inhibited subsystems */
-+ tmp_ss_mask = ss_mask;
-+ if (dst_root == &cgrp_dfl_root)
-+ tmp_ss_mask &= ~cgrp_dfl_root_inhibit_ss_mask;
-+
-+ ret = cgroup_populate_dir(&dst_root->cgrp, tmp_ss_mask);
-+ if (ret) {
-+ if (dst_root != &cgrp_dfl_root)
-+ return ret;
-+
-+ /*
-+ * Rebinding back to the default root is not allowed to
-+ * fail. Using both default and non-default roots should
-+ * be rare. Moving subsystems back and forth even more so.
-+ * Just warn about it and continue.
-+ */
-+ if (cgrp_dfl_root_visible) {
-+ pr_warn("failed to create files (%d) while rebinding 0x%x to default root\n",
-+ ret, ss_mask);
-+ pr_warn("you may retry by moving them to a different hierarchy and unbinding\n");
-+ }
-+ }
-+
-+ /*
-+ * Nothing can fail from this point on. Remove files for the
-+ * removed subsystems and rebind each subsystem.
-+ */
-+ for_each_subsys(ss, ssid)
-+ if (ss_mask & (1 << ssid))
-+ cgroup_clear_dir(&ss->root->cgrp, 1 << ssid);
-+
-+ for_each_subsys(ss, ssid) {
-+ struct cgroup_root *src_root;
-+ struct cgroup_subsys_state *css;
-+ struct css_set *cset;
-+
-+ if (!(ss_mask & (1 << ssid)))
-+ continue;
-+
-+ src_root = ss->root;
-+ css = cgroup_css(&src_root->cgrp, ss);
-+
-+ WARN_ON(!css || cgroup_css(&dst_root->cgrp, ss));
-+
-+ RCU_INIT_POINTER(src_root->cgrp.subsys[ssid], NULL);
-+ rcu_assign_pointer(dst_root->cgrp.subsys[ssid], css);
-+ ss->root = dst_root;
-+ css->cgroup = &dst_root->cgrp;
-+
-+ down_write(&css_set_rwsem);
-+ hash_for_each(css_set_table, i, cset, hlist)
-+ list_move_tail(&cset->e_cset_node[ss->id],
-+ &dst_root->cgrp.e_csets[ss->id]);
-+ up_write(&css_set_rwsem);
-+
-+ src_root->subsys_mask &= ~(1 << ssid);
-+ src_root->cgrp.subtree_control &= ~(1 << ssid);
-+ cgroup_refresh_child_subsys_mask(&src_root->cgrp);
-+
-+ /* default hierarchy doesn't enable controllers by default */
-+ dst_root->subsys_mask |= 1 << ssid;
-+ if (dst_root != &cgrp_dfl_root) {
-+ dst_root->cgrp.subtree_control |= 1 << ssid;
-+ cgroup_refresh_child_subsys_mask(&dst_root->cgrp);
-+ }
-+
-+ if (ss->bind)
-+ ss->bind(css);
-+ }
-+
-+ kernfs_activate(dst_root->cgrp.kn);
-+ return 0;
-+}
-+
-+static int cgroup_show_options(struct seq_file *seq,
-+ struct kernfs_root *kf_root)
-+{
-+ struct cgroup_root *root = cgroup_root_from_kf(kf_root);
-+ struct cgroup_subsys *ss;
-+ int ssid;
-+
-+ for_each_subsys(ss, ssid)
-+ if (root->subsys_mask & (1 << ssid))
-+ seq_show_option(seq, ss->name, NULL);
-+ if (root->flags & CGRP_ROOT_NOPREFIX)
-+ seq_puts(seq, ",noprefix");
-+ if (root->flags & CGRP_ROOT_XATTR)
-+ seq_puts(seq, ",xattr");
-+
-+ spin_lock(&release_agent_path_lock);
-+ if (strlen(root->release_agent_path))
-+ seq_show_option(seq, "release_agent",
-+ root->release_agent_path);
-+ spin_unlock(&release_agent_path_lock);
-+
-+ if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags))
-+ seq_puts(seq, ",clone_children");
-+ if (strlen(root->name))
-+ seq_show_option(seq, "name", root->name);
-+ return 0;
-+}
-+
-+struct cgroup_sb_opts {
-+ unsigned int subsys_mask;
-+ unsigned int flags;
-+ char *release_agent;
-+ bool cpuset_clone_children;
-+ char *name;
-+ /* User explicitly requested empty subsystem */
-+ bool none;
-+};
-+
-+static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
-+{
-+ char *token, *o = data;
-+ bool all_ss = false, one_ss = false;
-+ unsigned int mask = -1U;
-+ struct cgroup_subsys *ss;
-+ int nr_opts = 0;
-+ int i;
-+
-+#ifdef CONFIG_CPUSETS
-+ mask = ~(1U << cpuset_cgrp_id);
-+#endif
-+
-+ memset(opts, 0, sizeof(*opts));
-+
-+ while ((token = strsep(&o, ",")) != NULL) {
-+ nr_opts++;
-+
-+ if (!*token)
-+ return -EINVAL;
-+ if (!strcmp(token, "none")) {
-+ /* Explicitly have no subsystems */
-+ opts->none = true;
-+ continue;
-+ }
-+ if (!strcmp(token, "all")) {
-+ /* Mutually exclusive option 'all' + subsystem name */
-+ if (one_ss)
-+ return -EINVAL;
-+ all_ss = true;
-+ continue;
-+ }
-+ if (!strcmp(token, "__DEVEL__sane_behavior")) {
-+ opts->flags |= CGRP_ROOT_SANE_BEHAVIOR;
-+ continue;
-+ }
-+ if (!strcmp(token, "noprefix")) {
-+ opts->flags |= CGRP_ROOT_NOPREFIX;
-+ continue;
-+ }
-+ if (!strcmp(token, "clone_children")) {
-+ opts->cpuset_clone_children = true;
-+ continue;
-+ }
-+ if (!strcmp(token, "xattr")) {
-+ opts->flags |= CGRP_ROOT_XATTR;
-+ continue;
-+ }
-+ if (!strncmp(token, "release_agent=", 14)) {
-+ /* Specifying two release agents is forbidden */
-+ if (opts->release_agent)
-+ return -EINVAL;
-+ opts->release_agent =
-+ kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
-+ if (!opts->release_agent)
-+ return -ENOMEM;
-+ continue;
-+ }
-+ if (!strncmp(token, "name=", 5)) {
-+ const char *name = token + 5;
-+ /* Can't specify an empty name */
-+ if (!strlen(name))
-+ return -EINVAL;
-+ /* Must match [\w.-]+ */
-+ for (i = 0; i < strlen(name); i++) {
-+ char c = name[i];
-+ if (isalnum(c))
-+ continue;
-+ if ((c == '.') || (c == '-') || (c == '_'))
-+ continue;
-+ return -EINVAL;
-+ }
-+ /* Specifying two names is forbidden */
-+ if (opts->name)
-+ return -EINVAL;
-+ opts->name = kstrndup(name,
-+ MAX_CGROUP_ROOT_NAMELEN - 1,
-+ GFP_KERNEL);
-+ if (!opts->name)
-+ return -ENOMEM;
-+
-+ continue;
-+ }
-+
-+ for_each_subsys(ss, i) {
-+ if (strcmp(token, ss->name))
-+ continue;
-+ if (ss->disabled)
-+ continue;
-+
-+ /* Mutually exclusive option 'all' + subsystem name */
-+ if (all_ss)
-+ return -EINVAL;
-+ opts->subsys_mask |= (1 << i);
-+ one_ss = true;
-+
-+ break;
-+ }
-+ if (i == CGROUP_SUBSYS_COUNT)
-+ return -ENOENT;
-+ }
-+
-+ if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) {
-+ pr_warn("sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n");
-+ if (nr_opts != 1) {
-+ pr_err("sane_behavior: no other mount options allowed\n");
-+ return -EINVAL;
-+ }
-+ return 0;
-+ }
-+
-+ /*
-+ * If the 'all' option was specified select all the subsystems,
-+ * otherwise if 'none', 'name=' and a subsystem name options were
-+ * not specified, let's default to 'all'
-+ */
-+ if (all_ss || (!one_ss && !opts->none && !opts->name))
-+ for_each_subsys(ss, i)
-+ if (!ss->disabled)
-+ opts->subsys_mask |= (1 << i);
-+
-+ /*
-+ * We either have to specify by name or by subsystems. (So all
-+ * empty hierarchies must have a name).
-+ */
-+ if (!opts->subsys_mask && !opts->name)
-+ return -EINVAL;
-+
-+ /*
-+ * Option noprefix was introduced just for backward compatibility
-+ * with the old cpuset, so we allow noprefix only if mounting just
-+ * the cpuset subsystem.
-+ */
-+ if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
-+ return -EINVAL;
-+
-+ /* Can't specify "none" and some subsystems */
-+ if (opts->subsys_mask && opts->none)
-+ return -EINVAL;
-+
-+ return 0;
-+}
-+
-+static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
-+{
-+ int ret = 0;
-+ struct cgroup_root *root = cgroup_root_from_kf(kf_root);
-+ struct cgroup_sb_opts opts;
-+ unsigned int added_mask, removed_mask;
-+
-+ if (root == &cgrp_dfl_root) {
-+ pr_err("remount is not allowed\n");
-+ return -EINVAL;
-+ }
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ /* See what subsystems are wanted */
-+ ret = parse_cgroupfs_options(data, &opts);
-+ if (ret)
-+ goto out_unlock;
-+
-+ if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
-+ pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n",
-+ task_tgid_nr(current), current->comm);
-+
-+ added_mask = opts.subsys_mask & ~root->subsys_mask;
-+ removed_mask = root->subsys_mask & ~opts.subsys_mask;
-+
-+ /* Don't allow flags or name to change at remount */
-+ if ((opts.flags ^ root->flags) ||
-+ (opts.name && strcmp(opts.name, root->name))) {
-+ pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n",
-+ opts.flags, opts.name ?: "", root->flags, root->name);
-+ ret = -EINVAL;
-+ goto out_unlock;
-+ }
-+
-+ /* remounting is not allowed for populated hierarchies */
-+ if (!list_empty(&root->cgrp.self.children)) {
-+ ret = -EBUSY;
-+ goto out_unlock;
-+ }
-+
-+ ret = rebind_subsystems(root, added_mask);
-+ if (ret)
-+ goto out_unlock;
-+
-+ rebind_subsystems(&cgrp_dfl_root, removed_mask);
-+
-+ if (opts.release_agent) {
-+ spin_lock(&release_agent_path_lock);
-+ strcpy(root->release_agent_path, opts.release_agent);
-+ spin_unlock(&release_agent_path_lock);
-+ }
-+ out_unlock:
-+ kfree(opts.release_agent);
-+ kfree(opts.name);
-+ mutex_unlock(&cgroup_mutex);
-+ return ret;
-+}
-+
-+/*
-+ * To reduce the fork() overhead for systems that are not actually using
-+ * their cgroups capability, we don't maintain the lists running through
-+ * each css_set to its tasks until we see the list actually used - in other
-+ * words after the first mount.
-+ */
-+static bool use_task_css_set_links __read_mostly;
-+
-+static void cgroup_enable_task_cg_lists(void)
-+{
-+ struct task_struct *p, *g;
-+
-+ down_write(&css_set_rwsem);
-+
-+ if (use_task_css_set_links)
-+ goto out_unlock;
-+
-+ use_task_css_set_links = true;
-+
-+ /*
-+ * We need tasklist_lock because RCU is not safe against
-+ * while_each_thread(). Besides, a forking task that has passed
-+ * cgroup_post_fork() without seeing use_task_css_set_links = 1
-+ * is not guaranteed to have its child immediately visible in the
-+ * tasklist if we walk through it with RCU.
-+ */
-+ read_lock(&tasklist_lock);
-+ do_each_thread(g, p) {
-+ WARN_ON_ONCE(!list_empty(&p->cg_list) ||
-+ task_css_set(p) != &init_css_set);
-+
-+ /*
-+ * We should check if the process is exiting, otherwise
-+ * it will race with cgroup_exit() in that the list
-+ * entry won't be deleted though the process has exited.
-+ * Do it while holding siglock so that we don't end up
-+ * racing against cgroup_exit().
-+ */
-+ spin_lock_irq(&p->sighand->siglock);
-+ if (!(p->flags & PF_EXITING)) {
-+ struct css_set *cset = task_css_set(p);
-+
-+ list_add(&p->cg_list, &cset->tasks);
-+ get_css_set(cset);
-+ }
-+ spin_unlock_irq(&p->sighand->siglock);
-+ } while_each_thread(g, p);
-+ read_unlock(&tasklist_lock);
-+out_unlock:
-+ up_write(&css_set_rwsem);
-+}
-+
-+static void init_cgroup_housekeeping(struct cgroup *cgrp)
-+{
-+ struct cgroup_subsys *ss;
-+ int ssid;
-+
-+ INIT_LIST_HEAD(&cgrp->self.sibling);
-+ INIT_LIST_HEAD(&cgrp->self.children);
-+ INIT_LIST_HEAD(&cgrp->cset_links);
-+ INIT_LIST_HEAD(&cgrp->pidlists);
-+ mutex_init(&cgrp->pidlist_mutex);
-+ cgrp->self.cgroup = cgrp;
-+ cgrp->self.flags |= CSS_ONLINE;
-+
-+ for_each_subsys(ss, ssid)
-+ INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
-+
-+ init_waitqueue_head(&cgrp->offline_waitq);
-+ INIT_WORK(&cgrp->release_agent_work, cgroup_release_agent);
-+}
-+
-+static void init_cgroup_root(struct cgroup_root *root,
-+ struct cgroup_sb_opts *opts)
-+{
-+ struct cgroup *cgrp = &root->cgrp;
-+
-+ INIT_LIST_HEAD(&root->root_list);
-+ atomic_set(&root->nr_cgrps, 1);
-+ cgrp->root = root;
-+ init_cgroup_housekeeping(cgrp);
-+ idr_init(&root->cgroup_idr);
-+
-+ root->flags = opts->flags;
-+ if (opts->release_agent)
-+ strcpy(root->release_agent_path, opts->release_agent);
-+ if (opts->name)
-+ strcpy(root->name, opts->name);
-+ if (opts->cpuset_clone_children)
-+ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
-+}
-+
-+static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask)
-+{
-+ LIST_HEAD(tmp_links);
-+ struct cgroup *root_cgrp = &root->cgrp;
-+ struct cftype *base_files;
-+ struct css_set *cset;
-+ int i, ret;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_NOWAIT);
-+ if (ret < 0)
-+ goto out;
-+ root_cgrp->id = ret;
-+
-+ ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 0,
-+ GFP_KERNEL);
-+ if (ret)
-+ goto out;
-+
-+ /*
-+ * We're accessing css_set_count without locking css_set_rwsem here,
-+ * but that's OK - it can only be increased by someone holding
-+ * cgroup_lock, and that's us. The worst that can happen is that we
-+ * have some link structures left over
-+ */
-+ ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
-+ if (ret)
-+ goto cancel_ref;
-+
-+ ret = cgroup_init_root_id(root);
-+ if (ret)
-+ goto cancel_ref;
-+
-+ root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops,
-+ KERNFS_ROOT_CREATE_DEACTIVATED,
-+ root_cgrp);
-+ if (IS_ERR(root->kf_root)) {
-+ ret = PTR_ERR(root->kf_root);
-+ goto exit_root_id;
-+ }
-+ root_cgrp->kn = root->kf_root->kn;
-+
-+ if (root == &cgrp_dfl_root)
-+ base_files = cgroup_dfl_base_files;
-+ else
-+ base_files = cgroup_legacy_base_files;
-+
-+ ret = cgroup_addrm_files(root_cgrp, base_files, true);
-+ if (ret)
-+ goto destroy_root;
-+
-+ ret = rebind_subsystems(root, ss_mask);
-+ if (ret)
-+ goto destroy_root;
-+
-+ /*
-+ * There must be no failure case after here, since rebinding takes
-+ * care of subsystems' refcounts, which are explicitly dropped in
-+ * the failure exit path.
-+ */
-+ list_add(&root->root_list, &cgroup_roots);
-+ cgroup_root_count++;
-+
-+ /*
-+ * Link the root cgroup in this hierarchy into all the css_set
-+ * objects.
-+ */
-+ down_write(&css_set_rwsem);
-+ hash_for_each(css_set_table, i, cset, hlist)
-+ link_css_set(&tmp_links, cset, root_cgrp);
-+ up_write(&css_set_rwsem);
-+
-+ BUG_ON(!list_empty(&root_cgrp->self.children));
-+ BUG_ON(atomic_read(&root->nr_cgrps) != 1);
-+
-+ kernfs_activate(root_cgrp->kn);
-+ ret = 0;
-+ goto out;
-+
-+destroy_root:
-+ kernfs_destroy_root(root->kf_root);
-+ root->kf_root = NULL;
-+exit_root_id:
-+ cgroup_exit_root_id(root);
-+cancel_ref:
-+ percpu_ref_exit(&root_cgrp->self.refcnt);
-+out:
-+ free_cgrp_cset_links(&tmp_links);
-+ return ret;
-+}
-+
-+static struct dentry *cgroup_mount(struct file_system_type *fs_type,
-+ int flags, const char *unused_dev_name,
-+ void *data)
-+{
-+ struct super_block *pinned_sb = NULL;
-+ struct cgroup_subsys *ss;
-+ struct cgroup_root *root;
-+ struct cgroup_sb_opts opts;
-+ struct dentry *dentry;
-+ int ret;
-+ int i;
-+ bool new_sb;
-+
-+ /*
-+ * The first time anyone tries to mount a cgroup, enable the list
-+ * linking each css_set to its tasks and fix up all existing tasks.
-+ */
-+ if (!use_task_css_set_links)
-+ cgroup_enable_task_cg_lists();
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ /* First find the desired set of subsystems */
-+ ret = parse_cgroupfs_options(data, &opts);
-+ if (ret)
-+ goto out_unlock;
-+
-+ /* look for a matching existing root */
-+ if (opts.flags & CGRP_ROOT_SANE_BEHAVIOR) {
-+ cgrp_dfl_root_visible = true;
-+ root = &cgrp_dfl_root;
-+ cgroup_get(&root->cgrp);
-+ ret = 0;
-+ goto out_unlock;
-+ }
-+
-+ /*
-+ * Destruction of cgroup root is asynchronous, so subsystems may
-+ * still be dying after the previous unmount. Let's drain the
-+ * dying subsystems. We just need to ensure that the ones
-+ * unmounted previously finish dying and don't care about new ones
-+ * starting. Testing ref liveliness is good enough.
-+ */
-+ for_each_subsys(ss, i) {
-+ if (!(opts.subsys_mask & (1 << i)) ||
-+ ss->root == &cgrp_dfl_root)
-+ continue;
-+
-+ if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) {
-+ mutex_unlock(&cgroup_mutex);
-+ msleep(10);
-+ ret = restart_syscall();
-+ goto out_free;
-+ }
-+ cgroup_put(&ss->root->cgrp);
-+ }
-+
-+ for_each_root(root) {
-+ bool name_match = false;
-+
-+ if (root == &cgrp_dfl_root)
-+ continue;
-+
-+ /*
-+ * If we asked for a name then it must match. Also, if
-+ * name matches but sybsys_mask doesn't, we should fail.
-+ * Remember whether name matched.
-+ */
-+ if (opts.name) {
-+ if (strcmp(opts.name, root->name))
-+ continue;
-+ name_match = true;
-+ }
-+
-+ /*
-+ * If we asked for subsystems (or explicitly for no
-+ * subsystems) then they must match.
-+ */
-+ if ((opts.subsys_mask || opts.none) &&
-+ (opts.subsys_mask != root->subsys_mask)) {
-+ if (!name_match)
-+ continue;
-+ ret = -EBUSY;
-+ goto out_unlock;
-+ }
-+
-+ if (root->flags ^ opts.flags)
-+ pr_warn("new mount options do not match the existing superblock, will be ignored\n");
-+
-+ /*
-+ * We want to reuse @root whose lifetime is governed by its
-+ * ->cgrp. Let's check whether @root is alive and keep it
-+ * that way. As cgroup_kill_sb() can happen anytime, we
-+ * want to block it by pinning the sb so that @root doesn't
-+ * get killed before mount is complete.
-+ *
-+ * With the sb pinned, tryget_live can reliably indicate
-+ * whether @root can be reused. If it's being killed,
-+ * drain it. We can use wait_queue for the wait but this
-+ * path is super cold. Let's just sleep a bit and retry.
-+ */
-+ pinned_sb = kernfs_pin_sb(root->kf_root, NULL);
-+ if (IS_ERR(pinned_sb) ||
-+ !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) {
-+ mutex_unlock(&cgroup_mutex);
-+ if (!IS_ERR_OR_NULL(pinned_sb))
-+ deactivate_super(pinned_sb);
-+ msleep(10);
-+ ret = restart_syscall();
-+ goto out_free;
-+ }
-+
-+ ret = 0;
-+ goto out_unlock;
-+ }
-+
-+ /*
-+ * No such thing, create a new one. name= matching without subsys
-+ * specification is allowed for already existing hierarchies but we
-+ * can't create new one without subsys specification.
-+ */
-+ if (!opts.subsys_mask && !opts.none) {
-+ ret = -EINVAL;
-+ goto out_unlock;
-+ }
-+
-+ root = kzalloc(sizeof(*root), GFP_KERNEL);
-+ if (!root) {
-+ ret = -ENOMEM;
-+ goto out_unlock;
-+ }
-+
-+ init_cgroup_root(root, &opts);
-+
-+ ret = cgroup_setup_root(root, opts.subsys_mask);
-+ if (ret)
-+ cgroup_free_root(root);
-+
-+out_unlock:
-+ mutex_unlock(&cgroup_mutex);
-+out_free:
-+ kfree(opts.release_agent);
-+ kfree(opts.name);
-+
-+ if (ret)
-+ return ERR_PTR(ret);
-+
-+ dentry = kernfs_mount(fs_type, flags, root->kf_root,
-+ CGROUP_SUPER_MAGIC, &new_sb);
-+ if (IS_ERR(dentry) || !new_sb)
-+ cgroup_put(&root->cgrp);
-+
-+ /*
-+ * If @pinned_sb, we're reusing an existing root and holding an
-+ * extra ref on its sb. Mount is complete. Put the extra ref.
-+ */
-+ if (pinned_sb) {
-+ WARN_ON(new_sb);
-+ deactivate_super(pinned_sb);
-+ }
-+
-+ return dentry;
-+}
-+
-+static void cgroup_kill_sb(struct super_block *sb)
-+{
-+ struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
-+ struct cgroup_root *root = cgroup_root_from_kf(kf_root);
-+
-+ /*
-+ * If @root doesn't have any mounts or children, start killing it.
-+ * This prevents new mounts by disabling percpu_ref_tryget_live().
-+ * cgroup_mount() may wait for @root's release.
-+ *
-+ * And don't kill the default root.
-+ */
-+ if (!list_empty(&root->cgrp.self.children) ||
-+ root == &cgrp_dfl_root)
-+ cgroup_put(&root->cgrp);
-+ else
-+ percpu_ref_kill(&root->cgrp.self.refcnt);
-+
-+ kernfs_kill_sb(sb);
-+}
-+
-+static struct file_system_type cgroup_fs_type = {
-+ .name = "cgroup",
-+ .mount = cgroup_mount,
-+ .kill_sb = cgroup_kill_sb,
-+};
-+
-+/**
-+ * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
-+ * @task: target task
-+ * @buf: the buffer to write the path into
-+ * @buflen: the length of the buffer
-+ *
-+ * Determine @task's cgroup on the first (the one with the lowest non-zero
-+ * hierarchy_id) cgroup hierarchy and copy its path into @buf. This
-+ * function grabs cgroup_mutex and shouldn't be used inside locks used by
-+ * cgroup controller callbacks.
-+ *
-+ * Return value is the same as kernfs_path().
-+ */
-+char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
-+{
-+ struct cgroup_root *root;
-+ struct cgroup *cgrp;
-+ int hierarchy_id = 1;
-+ char *path = NULL;
-+
-+ mutex_lock(&cgroup_mutex);
-+ down_read(&css_set_rwsem);
-+
-+ root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);
-+
-+ if (root) {
-+ cgrp = task_cgroup_from_root(task, root);
-+ path = cgroup_path(cgrp, buf, buflen);
-+ } else {
-+ /* if no hierarchy exists, everyone is in "/" */
-+ if (strlcpy(buf, "/", buflen) < buflen)
-+ path = buf;
-+ }
-+
-+ up_read(&css_set_rwsem);
-+ mutex_unlock(&cgroup_mutex);
-+ return path;
-+}
-+EXPORT_SYMBOL_GPL(task_cgroup_path);
-+
-+/* used to track tasks and other necessary states during migration */
-+struct cgroup_taskset {
-+ /* the src and dst cset list running through cset->mg_node */
-+ struct list_head src_csets;
-+ struct list_head dst_csets;
-+
-+ /*
-+ * Fields for cgroup_taskset_*() iteration.
-+ *
-+ * Before migration is committed, the target migration tasks are on
-+ * ->mg_tasks of the csets on ->src_csets. After, on ->mg_tasks of
-+ * the csets on ->dst_csets. ->csets point to either ->src_csets
-+ * or ->dst_csets depending on whether migration is committed.
-+ *
-+ * ->cur_csets and ->cur_task point to the current task position
-+ * during iteration.
-+ */
-+ struct list_head *csets;
-+ struct css_set *cur_cset;
-+ struct task_struct *cur_task;
-+};
-+
-+/**
-+ * cgroup_taskset_first - reset taskset and return the first task
-+ * @tset: taskset of interest
-+ *
-+ * @tset iteration is initialized and the first task is returned.
-+ */
-+struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset)
-+{
-+ tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
-+ tset->cur_task = NULL;
-+
-+ return cgroup_taskset_next(tset);
-+}
-+
-+/**
-+ * cgroup_taskset_next - iterate to the next task in taskset
-+ * @tset: taskset of interest
-+ *
-+ * Return the next task in @tset. Iteration must have been initialized
-+ * with cgroup_taskset_first().
-+ */
-+struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
-+{
-+ struct css_set *cset = tset->cur_cset;
-+ struct task_struct *task = tset->cur_task;
-+
-+ while (&cset->mg_node != tset->csets) {
-+ if (!task)
-+ task = list_first_entry(&cset->mg_tasks,
-+ struct task_struct, cg_list);
-+ else
-+ task = list_next_entry(task, cg_list);
-+
-+ if (&task->cg_list != &cset->mg_tasks) {
-+ tset->cur_cset = cset;
-+ tset->cur_task = task;
-+ return task;
-+ }
-+
-+ cset = list_next_entry(cset, mg_node);
-+ task = NULL;
-+ }
-+
-+ return NULL;
-+}
-+
-+/**
-+ * cgroup_task_migrate - move a task from one cgroup to another.
-+ * @old_cgrp: the cgroup @tsk is being migrated from
-+ * @tsk: the task being migrated
-+ * @new_cset: the new css_set @tsk is being attached to
-+ *
-+ * Must be called with cgroup_mutex, threadgroup and css_set_rwsem locked.
-+ */
-+static void cgroup_task_migrate(struct cgroup *old_cgrp,
-+ struct task_struct *tsk,
-+ struct css_set *new_cset)
-+{
-+ struct css_set *old_cset;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+ lockdep_assert_held(&css_set_rwsem);
-+
-+ /*
-+ * We are synchronized through threadgroup_lock() against PF_EXITING
-+ * setting such that we can't race against cgroup_exit() changing the
-+ * css_set to init_css_set and dropping the old one.
-+ */
-+ WARN_ON_ONCE(tsk->flags & PF_EXITING);
-+ old_cset = task_css_set(tsk);
-+
-+ get_css_set(new_cset);
-+ rcu_assign_pointer(tsk->cgroups, new_cset);
-+
-+ /*
-+ * Use move_tail so that cgroup_taskset_first() still returns the
-+ * leader after migration. This works because cgroup_migrate()
-+ * ensures that the dst_cset of the leader is the first on the
-+ * tset's dst_csets list.
-+ */
-+ list_move_tail(&tsk->cg_list, &new_cset->mg_tasks);
-+
-+ /*
-+ * We just gained a reference on old_cset by taking it from the
-+ * task. As trading it for new_cset is protected by cgroup_mutex,
-+ * we're safe to drop it here; it will be freed under RCU.
-+ */
-+ put_css_set_locked(old_cset);
-+}
-+
-+/**
-+ * cgroup_migrate_finish - cleanup after attach
-+ * @preloaded_csets: list of preloaded css_sets
-+ *
-+ * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See
-+ * those functions for details.
-+ */
-+static void cgroup_migrate_finish(struct list_head *preloaded_csets)
-+{
-+ struct css_set *cset, *tmp_cset;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ down_write(&css_set_rwsem);
-+ list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) {
-+ cset->mg_src_cgrp = NULL;
-+ cset->mg_dst_cset = NULL;
-+ list_del_init(&cset->mg_preload_node);
-+ put_css_set_locked(cset);
-+ }
-+ up_write(&css_set_rwsem);
-+}
-+
-+/**
-+ * cgroup_migrate_add_src - add a migration source css_set
-+ * @src_cset: the source css_set to add
-+ * @dst_cgrp: the destination cgroup
-+ * @preloaded_csets: list of preloaded css_sets
-+ *
-+ * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin
-+ * @src_cset and add it to @preloaded_csets, which should later be cleaned
-+ * up by cgroup_migrate_finish().
-+ *
-+ * This function may be called without holding threadgroup_lock even if the
-+ * target is a process. Threads may be created and destroyed but as long
-+ * as cgroup_mutex is not dropped, no new css_set can be put into play and
-+ * the preloaded css_sets are guaranteed to cover all migrations.
-+ */
-+static void cgroup_migrate_add_src(struct css_set *src_cset,
-+ struct cgroup *dst_cgrp,
-+ struct list_head *preloaded_csets)
-+{
-+ struct cgroup *src_cgrp;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+ lockdep_assert_held(&css_set_rwsem);
-+
-+ src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);
-+
-+ if (!list_empty(&src_cset->mg_preload_node))
-+ return;
-+
-+ WARN_ON(src_cset->mg_src_cgrp);
-+ WARN_ON(!list_empty(&src_cset->mg_tasks));
-+ WARN_ON(!list_empty(&src_cset->mg_node));
-+
-+ src_cset->mg_src_cgrp = src_cgrp;
-+ get_css_set(src_cset);
-+ list_add(&src_cset->mg_preload_node, preloaded_csets);
-+}
-+
-+/**
-+ * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
-+ * @dst_cgrp: the destination cgroup (may be %NULL)
-+ * @preloaded_csets: list of preloaded source css_sets
-+ *
-+ * Tasks are about to be moved to @dst_cgrp and all the source css_sets
-+ * have been preloaded to @preloaded_csets. This function looks up and
-+ * pins all destination css_sets, links each to its source, and append them
-+ * to @preloaded_csets. If @dst_cgrp is %NULL, the destination of each
-+ * source css_set is assumed to be its cgroup on the default hierarchy.
-+ *
-+ * This function must be called after cgroup_migrate_add_src() has been
-+ * called on each migration source css_set. After migration is performed
-+ * using cgroup_migrate(), cgroup_migrate_finish() must be called on
-+ * @preloaded_csets.
-+ */
-+static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp,
-+ struct list_head *preloaded_csets)
-+{
-+ LIST_HEAD(csets);
-+ struct css_set *src_cset, *tmp_cset;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ /*
-+ * Except for the root, child_subsys_mask must be zero for a cgroup
-+ * with tasks so that child cgroups don't compete against tasks.
-+ */
-+ if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) &&
-+ dst_cgrp->child_subsys_mask)
-+ return -EBUSY;
-+
-+ /* look up the dst cset for each src cset and link it to src */
-+ list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) {
-+ struct css_set *dst_cset;
-+
-+ dst_cset = find_css_set(src_cset,
-+ dst_cgrp ?: src_cset->dfl_cgrp);
-+ if (!dst_cset)
-+ goto err;
-+
-+ WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
-+
-+ /*
-+ * If src cset equals dst, it's noop. Drop the src.
-+ * cgroup_migrate() will skip the cset too. Note that we
-+ * can't handle src == dst as some nodes are used by both.
-+ */
-+ if (src_cset == dst_cset) {
-+ src_cset->mg_src_cgrp = NULL;
-+ list_del_init(&src_cset->mg_preload_node);
-+ put_css_set(src_cset);
-+ put_css_set(dst_cset);
-+ continue;
-+ }
-+
-+ src_cset->mg_dst_cset = dst_cset;
-+
-+ if (list_empty(&dst_cset->mg_preload_node))
-+ list_add(&dst_cset->mg_preload_node, &csets);
-+ else
-+ put_css_set(dst_cset);
-+ }
-+
-+ list_splice_tail(&csets, preloaded_csets);
-+ return 0;
-+err:
-+ cgroup_migrate_finish(&csets);
-+ return -ENOMEM;
-+}
-+
-+/**
-+ * cgroup_migrate - migrate a process or task to a cgroup
-+ * @cgrp: the destination cgroup
-+ * @leader: the leader of the process or the task to migrate
-+ * @threadgroup: whether @leader points to the whole process or a single task
-+ *
-+ * Migrate a process or task denoted by @leader to @cgrp. If migrating a
-+ * process, the caller must be holding threadgroup_lock of @leader. The
-+ * caller is also responsible for invoking cgroup_migrate_add_src() and
-+ * cgroup_migrate_prepare_dst() on the targets before invoking this
-+ * function and following up with cgroup_migrate_finish().
-+ *
-+ * As long as a controller's ->can_attach() doesn't fail, this function is
-+ * guaranteed to succeed. This means that, excluding ->can_attach()
-+ * failure, when migrating multiple targets, the success or failure can be
-+ * decided for all targets by invoking group_migrate_prepare_dst() before
-+ * actually starting migrating.
-+ */
-+static int cgroup_migrate(struct cgroup *cgrp, struct task_struct *leader,
-+ bool threadgroup)
-+{
-+ struct cgroup_taskset tset = {
-+ .src_csets = LIST_HEAD_INIT(tset.src_csets),
-+ .dst_csets = LIST_HEAD_INIT(tset.dst_csets),
-+ .csets = &tset.src_csets,
-+ };
-+ struct cgroup_subsys_state *css, *failed_css = NULL;
-+ struct css_set *cset, *tmp_cset;
-+ struct task_struct *task, *tmp_task;
-+ int i, ret;
-+
-+ /*
-+ * Prevent freeing of tasks while we take a snapshot. Tasks that are
-+ * already PF_EXITING could be freed from underneath us unless we
-+ * take an rcu_read_lock.
-+ */
-+ down_write(&css_set_rwsem);
-+ rcu_read_lock();
-+ task = leader;
-+ do {
-+ /* @task either already exited or can't exit until the end */
-+ if (task->flags & PF_EXITING)
-+ goto next;
-+
-+ /* leave @task alone if post_fork() hasn't linked it yet */
-+ if (list_empty(&task->cg_list))
-+ goto next;
-+
-+ cset = task_css_set(task);
-+ if (!cset->mg_src_cgrp)
-+ goto next;
-+
-+ /*
-+ * cgroup_taskset_first() must always return the leader.
-+ * Take care to avoid disturbing the ordering.
-+ */
-+ list_move_tail(&task->cg_list, &cset->mg_tasks);
-+ if (list_empty(&cset->mg_node))
-+ list_add_tail(&cset->mg_node, &tset.src_csets);
-+ if (list_empty(&cset->mg_dst_cset->mg_node))
-+ list_move_tail(&cset->mg_dst_cset->mg_node,
-+ &tset.dst_csets);
-+ next:
-+ if (!threadgroup)
-+ break;
-+ } while_each_thread(leader, task);
-+ rcu_read_unlock();
-+ up_write(&css_set_rwsem);
-+
-+ /* methods shouldn't be called if no task is actually migrating */
-+ if (list_empty(&tset.src_csets))
-+ return 0;
-+
-+ /* check that we can legitimately attach to the cgroup */
-+ for_each_e_css(css, i, cgrp) {
-+ if (css->ss->can_attach) {
-+ ret = css->ss->can_attach(css, &tset);
-+ if (ret) {
-+ failed_css = css;
-+ goto out_cancel_attach;
-+ }
-+ }
-+ }
-+
-+ /*
-+ * Now that we're guaranteed success, proceed to move all tasks to
-+ * the new cgroup. There are no failure cases after here, so this
-+ * is the commit point.
-+ */
-+ down_write(&css_set_rwsem);
-+ list_for_each_entry(cset, &tset.src_csets, mg_node) {
-+ list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list)
-+ cgroup_task_migrate(cset->mg_src_cgrp, task,
-+ cset->mg_dst_cset);
-+ }
-+ up_write(&css_set_rwsem);
-+
-+ /*
-+ * Migration is committed, all target tasks are now on dst_csets.
-+ * Nothing is sensitive to fork() after this point. Notify
-+ * controllers that migration is complete.
-+ */
-+ tset.csets = &tset.dst_csets;
-+
-+ for_each_e_css(css, i, cgrp)
-+ if (css->ss->attach)
-+ css->ss->attach(css, &tset);
-+
-+ ret = 0;
-+ goto out_release_tset;
-+
-+out_cancel_attach:
-+ for_each_e_css(css, i, cgrp) {
-+ if (css == failed_css)
-+ break;
-+ if (css->ss->cancel_attach)
-+ css->ss->cancel_attach(css, &tset);
-+ }
-+out_release_tset:
-+ down_write(&css_set_rwsem);
-+ list_splice_init(&tset.dst_csets, &tset.src_csets);
-+ list_for_each_entry_safe(cset, tmp_cset, &tset.src_csets, mg_node) {
-+ list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
-+ list_del_init(&cset->mg_node);
-+ }
-+ up_write(&css_set_rwsem);
-+ return ret;
-+}
-+
-+/**
-+ * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
-+ * @dst_cgrp: the cgroup to attach to
-+ * @leader: the task or the leader of the threadgroup to be attached
-+ * @threadgroup: attach the whole threadgroup?
-+ *
-+ * Call holding cgroup_mutex and threadgroup_lock of @leader.
-+ */
-+static int cgroup_attach_task(struct cgroup *dst_cgrp,
-+ struct task_struct *leader, bool threadgroup)
-+{
-+ LIST_HEAD(preloaded_csets);
-+ struct task_struct *task;
-+ int ret;
-+
-+ /* look up all src csets */
-+ down_read(&css_set_rwsem);
-+ rcu_read_lock();
-+ task = leader;
-+ do {
-+ cgroup_migrate_add_src(task_css_set(task), dst_cgrp,
-+ &preloaded_csets);
-+ if (!threadgroup)
-+ break;
-+ } while_each_thread(leader, task);
-+ rcu_read_unlock();
-+ up_read(&css_set_rwsem);
-+
-+ /* prepare dst csets and commit */
-+ ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets);
-+ if (!ret)
-+ ret = cgroup_migrate(dst_cgrp, leader, threadgroup);
-+
-+ cgroup_migrate_finish(&preloaded_csets);
-+ return ret;
-+}
-+
-+/*
-+ * Find the task_struct of the task to attach by vpid and pass it along to the
-+ * function to attach either it or all tasks in its threadgroup. Will lock
-+ * cgroup_mutex and threadgroup.
-+ */
-+static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
-+ size_t nbytes, loff_t off, bool threadgroup)
-+{
-+ struct task_struct *tsk;
-+ const struct cred *cred = current_cred(), *tcred;
-+ struct cgroup *cgrp;
-+ pid_t pid;
-+ int ret;
-+
-+ if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
-+ return -EINVAL;
-+
-+ cgrp = cgroup_kn_lock_live(of->kn);
-+ if (!cgrp)
-+ return -ENODEV;
-+
-+retry_find_task:
-+ rcu_read_lock();
-+ if (pid) {
-+ tsk = find_task_by_vpid(pid);
-+ if (!tsk) {
-+ rcu_read_unlock();
-+ ret = -ESRCH;
-+ goto out_unlock_cgroup;
-+ }
-+ /*
-+ * even if we're attaching all tasks in the thread group, we
-+ * only need to check permissions on one of them.
-+ */
-+ tcred = __task_cred(tsk);
-+ if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
-+ !uid_eq(cred->euid, tcred->uid) &&
-+ !uid_eq(cred->euid, tcred->suid)) {
-+ rcu_read_unlock();
-+ ret = -EACCES;
-+ goto out_unlock_cgroup;
-+ }
-+ } else
-+ tsk = current;
-+
-+ if (threadgroup)
-+ tsk = tsk->group_leader;
-+
-+ /*
-+ * Workqueue threads may acquire PF_NO_SETAFFINITY and become
-+ * trapped in a cpuset, or RT worker may be born in a cgroup
-+ * with no rt_runtime allocated. Just say no.
-+ */
-+ if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
-+ ret = -EINVAL;
-+ rcu_read_unlock();
-+ goto out_unlock_cgroup;
-+ }
-+
-+ get_task_struct(tsk);
-+ rcu_read_unlock();
-+
-+ threadgroup_lock(tsk);
-+ if (threadgroup) {
-+ if (!thread_group_leader(tsk)) {
-+ /*
-+ * a race with de_thread from another thread's exec()
-+ * may strip us of our leadership, if this happens,
-+ * there is no choice but to throw this task away and
-+ * try again; this is
-+ * "double-double-toil-and-trouble-check locking".
-+ */
-+ threadgroup_unlock(tsk);
-+ put_task_struct(tsk);
-+ goto retry_find_task;
-+ }
-+ }
-+
-+ ret = cgroup_attach_task(cgrp, tsk, threadgroup);
-+
-+ threadgroup_unlock(tsk);
-+
-+ put_task_struct(tsk);
-+out_unlock_cgroup:
-+ cgroup_kn_unlock(of->kn);
-+ return ret ?: nbytes;
-+}
-+
-+/**
-+ * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
-+ * @from: attach to all cgroups of a given task
-+ * @tsk: the task to be attached
-+ */
-+int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
-+{
-+ struct cgroup_root *root;
-+ int retval = 0;
-+
-+ mutex_lock(&cgroup_mutex);
-+ for_each_root(root) {
-+ struct cgroup *from_cgrp;
-+
-+ if (root == &cgrp_dfl_root)
-+ continue;
-+
-+ down_read(&css_set_rwsem);
-+ from_cgrp = task_cgroup_from_root(from, root);
-+ up_read(&css_set_rwsem);
-+
-+ retval = cgroup_attach_task(from_cgrp, tsk, false);
-+ if (retval)
-+ break;
-+ }
-+ mutex_unlock(&cgroup_mutex);
-+
-+ return retval;
-+}
-+EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
-+
-+static ssize_t cgroup_tasks_write(struct kernfs_open_file *of,
-+ char *buf, size_t nbytes, loff_t off)
-+{
-+ return __cgroup_procs_write(of, buf, nbytes, off, false);
-+}
-+
-+static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
-+ char *buf, size_t nbytes, loff_t off)
-+{
-+ return __cgroup_procs_write(of, buf, nbytes, off, true);
-+}
-+
-+static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
-+ char *buf, size_t nbytes, loff_t off)
-+{
-+ struct cgroup *cgrp;
-+
-+ BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
-+
-+ cgrp = cgroup_kn_lock_live(of->kn);
-+ if (!cgrp)
-+ return -ENODEV;
-+ spin_lock(&release_agent_path_lock);
-+ strlcpy(cgrp->root->release_agent_path, strstrip(buf),
-+ sizeof(cgrp->root->release_agent_path));
-+ spin_unlock(&release_agent_path_lock);
-+ cgroup_kn_unlock(of->kn);
-+ return nbytes;
-+}
-+
-+static int cgroup_release_agent_show(struct seq_file *seq, void *v)
-+{
-+ struct cgroup *cgrp = seq_css(seq)->cgroup;
-+
-+ spin_lock(&release_agent_path_lock);
-+ seq_puts(seq, cgrp->root->release_agent_path);
-+ spin_unlock(&release_agent_path_lock);
-+ seq_putc(seq, '\n');
-+ return 0;
-+}
-+
-+static int cgroup_sane_behavior_show(struct seq_file *seq, void *v)
-+{
-+ seq_puts(seq, "0\n");
-+ return 0;
-+}
-+
-+static void cgroup_print_ss_mask(struct seq_file *seq, unsigned int ss_mask)
-+{
-+ struct cgroup_subsys *ss;
-+ bool printed = false;
-+ int ssid;
-+
-+ for_each_subsys(ss, ssid) {
-+ if (ss_mask & (1 << ssid)) {
-+ if (printed)
-+ seq_putc(seq, ' ');
-+ seq_printf(seq, "%s", ss->name);
-+ printed = true;
-+ }
-+ }
-+ if (printed)
-+ seq_putc(seq, '\n');
-+}
-+
-+/* show controllers which are currently attached to the default hierarchy */
-+static int cgroup_root_controllers_show(struct seq_file *seq, void *v)
-+{
-+ struct cgroup *cgrp = seq_css(seq)->cgroup;
-+
-+ cgroup_print_ss_mask(seq, cgrp->root->subsys_mask &
-+ ~cgrp_dfl_root_inhibit_ss_mask);
-+ return 0;
-+}
-+
-+/* show controllers which are enabled from the parent */
-+static int cgroup_controllers_show(struct seq_file *seq, void *v)
-+{
-+ struct cgroup *cgrp = seq_css(seq)->cgroup;
-+
-+ cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->subtree_control);
-+ return 0;
-+}
-+
-+/* show controllers which are enabled for a given cgroup's children */
-+static int cgroup_subtree_control_show(struct seq_file *seq, void *v)
-+{
-+ struct cgroup *cgrp = seq_css(seq)->cgroup;
-+
-+ cgroup_print_ss_mask(seq, cgrp->subtree_control);
-+ return 0;
-+}
-+
-+/**
-+ * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
-+ * @cgrp: root of the subtree to update csses for
-+ *
-+ * @cgrp's child_subsys_mask has changed and its subtree's (self excluded)
-+ * css associations need to be updated accordingly. This function looks up
-+ * all css_sets which are attached to the subtree, creates the matching
-+ * updated css_sets and migrates the tasks to the new ones.
-+ */
-+static int cgroup_update_dfl_csses(struct cgroup *cgrp)
-+{
-+ LIST_HEAD(preloaded_csets);
-+ struct cgroup_subsys_state *css;
-+ struct css_set *src_cset;
-+ int ret;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ /* look up all csses currently attached to @cgrp's subtree */
-+ down_read(&css_set_rwsem);
-+ css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) {
-+ struct cgrp_cset_link *link;
-+
-+ /* self is not affected by child_subsys_mask change */
-+ if (css->cgroup == cgrp)
-+ continue;
-+
-+ list_for_each_entry(link, &css->cgroup->cset_links, cset_link)
-+ cgroup_migrate_add_src(link->cset, cgrp,
-+ &preloaded_csets);
-+ }
-+ up_read(&css_set_rwsem);
-+
-+ /* NULL dst indicates self on default hierarchy */
-+ ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets);
-+ if (ret)
-+ goto out_finish;
-+
-+ list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) {
-+ struct task_struct *last_task = NULL, *task;
-+
-+ /* src_csets precede dst_csets, break on the first dst_cset */
-+ if (!src_cset->mg_src_cgrp)
-+ break;
-+
-+ /*
-+ * All tasks in src_cset need to be migrated to the
-+ * matching dst_cset. Empty it process by process. We
-+ * walk tasks but migrate processes. The leader might even
-+ * belong to a different cset but such src_cset would also
-+ * be among the target src_csets because the default
-+ * hierarchy enforces per-process membership.
-+ */
-+ while (true) {
-+ down_read(&css_set_rwsem);
-+ task = list_first_entry_or_null(&src_cset->tasks,
-+ struct task_struct, cg_list);
-+ if (task) {
-+ task = task->group_leader;
-+ WARN_ON_ONCE(!task_css_set(task)->mg_src_cgrp);
-+ get_task_struct(task);
-+ }
-+ up_read(&css_set_rwsem);
-+
-+ if (!task)
-+ break;
-+
-+ /* guard against possible infinite loop */
-+ if (WARN(last_task == task,
-+ "cgroup: update_dfl_csses failed to make progress, aborting in inconsistent state\n"))
-+ goto out_finish;
-+ last_task = task;
-+
-+ threadgroup_lock(task);
-+ /* raced against de_thread() from another thread? */
-+ if (!thread_group_leader(task)) {
-+ threadgroup_unlock(task);
-+ put_task_struct(task);
-+ continue;
-+ }
-+
-+ ret = cgroup_migrate(src_cset->dfl_cgrp, task, true);
-+
-+ threadgroup_unlock(task);
-+ put_task_struct(task);
-+
-+ if (WARN(ret, "cgroup: failed to update controllers for the default hierarchy (%d), further operations may crash or hang\n", ret))
-+ goto out_finish;
-+ }
-+ }
-+
-+out_finish:
-+ cgroup_migrate_finish(&preloaded_csets);
-+ return ret;
-+}
-+
-+/* change the enabled child controllers for a cgroup in the default hierarchy */
-+static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
-+ char *buf, size_t nbytes,
-+ loff_t off)
-+{
-+ unsigned int enable = 0, disable = 0;
-+ unsigned int css_enable, css_disable, old_sc, new_sc, old_ss, new_ss;
-+ struct cgroup *cgrp, *child;
-+ struct cgroup_subsys *ss;
-+ char *tok;
-+ int ssid, ret;
-+
-+ /*
-+ * Parse input - space separated list of subsystem names prefixed
-+ * with either + or -.
-+ */
-+ buf = strstrip(buf);
-+ while ((tok = strsep(&buf, " "))) {
-+ if (tok[0] == '\0')
-+ continue;
-+ for_each_subsys(ss, ssid) {
-+ if (ss->disabled || strcmp(tok + 1, ss->name) ||
-+ ((1 << ss->id) & cgrp_dfl_root_inhibit_ss_mask))
-+ continue;
-+
-+ if (*tok == '+') {
-+ enable |= 1 << ssid;
-+ disable &= ~(1 << ssid);
-+ } else if (*tok == '-') {
-+ disable |= 1 << ssid;
-+ enable &= ~(1 << ssid);
-+ } else {
-+ return -EINVAL;
-+ }
-+ break;
-+ }
-+ if (ssid == CGROUP_SUBSYS_COUNT)
-+ return -EINVAL;
-+ }
-+
-+ cgrp = cgroup_kn_lock_live(of->kn);
-+ if (!cgrp)
-+ return -ENODEV;
-+
-+ for_each_subsys(ss, ssid) {
-+ if (enable & (1 << ssid)) {
-+ if (cgrp->subtree_control & (1 << ssid)) {
-+ enable &= ~(1 << ssid);
-+ continue;
-+ }
-+
-+ /* unavailable or not enabled on the parent? */
-+ if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) ||
-+ (cgroup_parent(cgrp) &&
-+ !(cgroup_parent(cgrp)->subtree_control & (1 << ssid)))) {
-+ ret = -ENOENT;
-+ goto out_unlock;
-+ }
-+ } else if (disable & (1 << ssid)) {
-+ if (!(cgrp->subtree_control & (1 << ssid))) {
-+ disable &= ~(1 << ssid);
-+ continue;
-+ }
-+
-+ /* a child has it enabled? */
-+ cgroup_for_each_live_child(child, cgrp) {
-+ if (child->subtree_control & (1 << ssid)) {
-+ ret = -EBUSY;
-+ goto out_unlock;
-+ }
-+ }
-+ }
-+ }
-+
-+ if (!enable && !disable) {
-+ ret = 0;
-+ goto out_unlock;
-+ }
-+
-+ /*
-+ * Except for the root, subtree_control must be zero for a cgroup
-+ * with tasks so that child cgroups don't compete against tasks.
-+ */
-+ if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) {
-+ ret = -EBUSY;
-+ goto out_unlock;
-+ }
-+
-+ /*
-+ * Update subsys masks and calculate what needs to be done. More
-+ * subsystems than specified may need to be enabled or disabled
-+ * depending on subsystem dependencies.
-+ */
-+ old_sc = cgrp->subtree_control;
-+ old_ss = cgrp->child_subsys_mask;
-+ new_sc = (old_sc | enable) & ~disable;
-+ new_ss = cgroup_calc_child_subsys_mask(cgrp, new_sc);
-+
-+ css_enable = ~old_ss & new_ss;
-+ css_disable = old_ss & ~new_ss;
-+ enable |= css_enable;
-+ disable |= css_disable;
-+
-+ /*
-+ * Because css offlining is asynchronous, userland might try to
-+ * re-enable the same controller while the previous instance is
-+ * still around. In such cases, wait till it's gone using
-+ * offline_waitq.
-+ */
-+ for_each_subsys(ss, ssid) {
-+ if (!(css_enable & (1 << ssid)))
-+ continue;
-+
-+ cgroup_for_each_live_child(child, cgrp) {
-+ DEFINE_WAIT(wait);
-+
-+ if (!cgroup_css(child, ss))
-+ continue;
-+
-+ cgroup_get(child);
-+ prepare_to_wait(&child->offline_waitq, &wait,
-+ TASK_UNINTERRUPTIBLE);
-+ cgroup_kn_unlock(of->kn);
-+ schedule();
-+ finish_wait(&child->offline_waitq, &wait);
-+ cgroup_put(child);
-+
-+ return restart_syscall();
-+ }
-+ }
-+
-+ cgrp->subtree_control = new_sc;
-+ cgrp->child_subsys_mask = new_ss;
-+
-+ /*
-+ * Create new csses or make the existing ones visible. A css is
-+ * created invisible if it's being implicitly enabled through
-+ * dependency. An invisible css is made visible when the userland
-+ * explicitly enables it.
-+ */
-+ for_each_subsys(ss, ssid) {
-+ if (!(enable & (1 << ssid)))
-+ continue;
-+
-+ cgroup_for_each_live_child(child, cgrp) {
-+ if (css_enable & (1 << ssid))
-+ ret = create_css(child, ss,
-+ cgrp->subtree_control & (1 << ssid));
-+ else
-+ ret = cgroup_populate_dir(child, 1 << ssid);
-+ if (ret)
-+ goto err_undo_css;
-+ }
-+ }
-+
-+ /*
-+ * At this point, cgroup_e_css() results reflect the new csses
-+ * making the following cgroup_update_dfl_csses() properly update
-+ * css associations of all tasks in the subtree.
-+ */
-+ ret = cgroup_update_dfl_csses(cgrp);
-+ if (ret)
-+ goto err_undo_css;
-+
-+ /*
-+ * All tasks are migrated out of disabled csses. Kill or hide
-+ * them. A css is hidden when the userland requests it to be
-+ * disabled while other subsystems are still depending on it. The
-+ * css must not actively control resources and be in the vanilla
-+ * state if it's made visible again later. Controllers which may
-+ * be depended upon should provide ->css_reset() for this purpose.
-+ */
-+ for_each_subsys(ss, ssid) {
-+ if (!(disable & (1 << ssid)))
-+ continue;
-+
-+ cgroup_for_each_live_child(child, cgrp) {
-+ struct cgroup_subsys_state *css = cgroup_css(child, ss);
-+
-+ if (css_disable & (1 << ssid)) {
-+ kill_css(css);
-+ } else {
-+ cgroup_clear_dir(child, 1 << ssid);
-+ if (ss->css_reset)
-+ ss->css_reset(css);
-+ }
-+ }
-+ }
-+
-+ /*
-+ * The effective csses of all the descendants (excluding @cgrp) may
-+ * have changed. Subsystems can optionally subscribe to this event
-+ * by implementing ->css_e_css_changed() which is invoked if any of
-+ * the effective csses seen from the css's cgroup may have changed.
-+ */
-+ for_each_subsys(ss, ssid) {
-+ struct cgroup_subsys_state *this_css = cgroup_css(cgrp, ss);
-+ struct cgroup_subsys_state *css;
-+
-+ if (!ss->css_e_css_changed || !this_css)
-+ continue;
-+
-+ css_for_each_descendant_pre(css, this_css)
-+ if (css != this_css)
-+ ss->css_e_css_changed(css);
-+ }
-+
-+ kernfs_activate(cgrp->kn);
-+ ret = 0;
-+out_unlock:
-+ cgroup_kn_unlock(of->kn);
-+ return ret ?: nbytes;
-+
-+err_undo_css:
-+ cgrp->subtree_control = old_sc;
-+ cgrp->child_subsys_mask = old_ss;
-+
-+ for_each_subsys(ss, ssid) {
-+ if (!(enable & (1 << ssid)))
-+ continue;
-+
-+ cgroup_for_each_live_child(child, cgrp) {
-+ struct cgroup_subsys_state *css = cgroup_css(child, ss);
-+
-+ if (!css)
-+ continue;
-+
-+ if (css_enable & (1 << ssid))
-+ kill_css(css);
-+ else
-+ cgroup_clear_dir(child, 1 << ssid);
-+ }
-+ }
-+ goto out_unlock;
-+}
-+
-+static int cgroup_populated_show(struct seq_file *seq, void *v)
-+{
-+ seq_printf(seq, "%d\n", (bool)seq_css(seq)->cgroup->populated_cnt);
-+ return 0;
-+}
-+
-+static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
-+ size_t nbytes, loff_t off)
-+{
-+ struct cgroup *cgrp = of->kn->parent->priv;
-+ struct cftype *cft = of->kn->priv;
-+ struct cgroup_subsys_state *css;
-+ int ret;
-+
-+ if (cft->write)
-+ return cft->write(of, buf, nbytes, off);
-+
-+ /*
-+ * kernfs guarantees that a file isn't deleted with operations in
-+ * flight, which means that the matching css is and stays alive and
-+ * doesn't need to be pinned. The RCU locking is not necessary
-+ * either. It's just for the convenience of using cgroup_css().
-+ */
-+ rcu_read_lock();
-+ css = cgroup_css(cgrp, cft->ss);
-+ rcu_read_unlock();
-+
-+ if (cft->write_u64) {
-+ unsigned long long v;
-+ ret = kstrtoull(buf, 0, &v);
-+ if (!ret)
-+ ret = cft->write_u64(css, cft, v);
-+ } else if (cft->write_s64) {
-+ long long v;
-+ ret = kstrtoll(buf, 0, &v);
-+ if (!ret)
-+ ret = cft->write_s64(css, cft, v);
-+ } else {
-+ ret = -EINVAL;
-+ }
-+
-+ return ret ?: nbytes;
-+}
-+
-+static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
-+{
-+ return seq_cft(seq)->seq_start(seq, ppos);
-+}
-+
-+static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
-+{
-+ return seq_cft(seq)->seq_next(seq, v, ppos);
-+}
-+
-+static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
-+{
-+ seq_cft(seq)->seq_stop(seq, v);
-+}
-+
-+static int cgroup_seqfile_show(struct seq_file *m, void *arg)
-+{
-+ struct cftype *cft = seq_cft(m);
-+ struct cgroup_subsys_state *css = seq_css(m);
-+
-+ if (cft->seq_show)
-+ return cft->seq_show(m, arg);
-+
-+ if (cft->read_u64)
-+ seq_printf(m, "%llu\n", cft->read_u64(css, cft));
-+ else if (cft->read_s64)
-+ seq_printf(m, "%lld\n", cft->read_s64(css, cft));
-+ else
-+ return -EINVAL;
-+ return 0;
-+}
-+
-+static struct kernfs_ops cgroup_kf_single_ops = {
-+ .atomic_write_len = PAGE_SIZE,
-+ .write = cgroup_file_write,
-+ .seq_show = cgroup_seqfile_show,
-+};
-+
-+static struct kernfs_ops cgroup_kf_ops = {
-+ .atomic_write_len = PAGE_SIZE,
-+ .write = cgroup_file_write,
-+ .seq_start = cgroup_seqfile_start,
-+ .seq_next = cgroup_seqfile_next,
-+ .seq_stop = cgroup_seqfile_stop,
-+ .seq_show = cgroup_seqfile_show,
-+};
-+
-+/*
-+ * cgroup_rename - Only allow simple rename of directories in place.
-+ */
-+static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent,
-+ const char *new_name_str)
-+{
-+ struct cgroup *cgrp = kn->priv;
-+ int ret;
-+
-+ if (kernfs_type(kn) != KERNFS_DIR)
-+ return -ENOTDIR;
-+ if (kn->parent != new_parent)
-+ return -EIO;
-+
-+ /*
-+ * This isn't a proper migration and its usefulness is very
-+ * limited. Disallow on the default hierarchy.
-+ */
-+ if (cgroup_on_dfl(cgrp))
-+ return -EPERM;
-+
-+ /*
-+ * We're gonna grab cgroup_mutex which nests outside kernfs
-+ * active_ref. kernfs_rename() doesn't require active_ref
-+ * protection. Break them before grabbing cgroup_mutex.
-+ */
-+ kernfs_break_active_protection(new_parent);
-+ kernfs_break_active_protection(kn);
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ ret = kernfs_rename(kn, new_parent, new_name_str);
-+
-+ mutex_unlock(&cgroup_mutex);
-+
-+ kernfs_unbreak_active_protection(kn);
-+ kernfs_unbreak_active_protection(new_parent);
-+ return ret;
-+}
-+
-+/* set uid and gid of cgroup dirs and files to that of the creator */
-+static int cgroup_kn_set_ugid(struct kernfs_node *kn)
-+{
-+ struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
-+ .ia_uid = current_fsuid(),
-+ .ia_gid = current_fsgid(), };
-+
-+ if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
-+ gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
-+ return 0;
-+
-+ return kernfs_setattr(kn, &iattr);
-+}
-+
-+static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
-+{
-+ char name[CGROUP_FILE_NAME_MAX];
-+ struct kernfs_node *kn;
-+ struct lock_class_key *key = NULL;
-+ int ret;
-+
-+#ifdef CONFIG_DEBUG_LOCK_ALLOC
-+ key = &cft->lockdep_key;
-+#endif
-+ kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
-+ cgroup_file_mode(cft), 0, cft->kf_ops, cft,
-+ NULL, key);
-+ if (IS_ERR(kn))
-+ return PTR_ERR(kn);
-+
-+ ret = cgroup_kn_set_ugid(kn);
-+ if (ret) {
-+ kernfs_remove(kn);
-+ return ret;
-+ }
-+
-+ if (cft->seq_show == cgroup_populated_show)
-+ cgrp->populated_kn = kn;
-+ return 0;
-+}
-+
-+/**
-+ * cgroup_addrm_files - add or remove files to a cgroup directory
-+ * @cgrp: the target cgroup
-+ * @cfts: array of cftypes to be added
-+ * @is_add: whether to add or remove
-+ *
-+ * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
-+ * For removals, this function never fails. If addition fails, this
-+ * function doesn't remove files already added. The caller is responsible
-+ * for cleaning up.
-+ */
-+static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
-+ bool is_add)
-+{
-+ struct cftype *cft;
-+ int ret;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ for (cft = cfts; cft->name[0] != '\0'; cft++) {
-+ /* does cft->flags tell us to skip this file on @cgrp? */
-+ if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
-+ continue;
-+ if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp))
-+ continue;
-+ if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
-+ continue;
-+ if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
-+ continue;
-+
-+ if (is_add) {
-+ ret = cgroup_add_file(cgrp, cft);
-+ if (ret) {
-+ pr_warn("%s: failed to add %s, err=%d\n",
-+ __func__, cft->name, ret);
-+ return ret;
-+ }
-+ } else {
-+ cgroup_rm_file(cgrp, cft);
-+ }
-+ }
-+ return 0;
-+}
-+
-+static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
-+{
-+ LIST_HEAD(pending);
-+ struct cgroup_subsys *ss = cfts[0].ss;
-+ struct cgroup *root = &ss->root->cgrp;
-+ struct cgroup_subsys_state *css;
-+ int ret = 0;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ /* add/rm files for all cgroups created before */
-+ css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
-+ struct cgroup *cgrp = css->cgroup;
-+
-+ if (cgroup_is_dead(cgrp))
-+ continue;
-+
-+ ret = cgroup_addrm_files(cgrp, cfts, is_add);
-+ if (ret)
-+ break;
-+ }
-+
-+ if (is_add && !ret)
-+ kernfs_activate(root->kn);
-+ return ret;
-+}
-+
-+static void cgroup_exit_cftypes(struct cftype *cfts)
-+{
-+ struct cftype *cft;
-+
-+ for (cft = cfts; cft->name[0] != '\0'; cft++) {
-+ /* free copy for custom atomic_write_len, see init_cftypes() */
-+ if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
-+ kfree(cft->kf_ops);
-+ cft->kf_ops = NULL;
-+ cft->ss = NULL;
-+
-+ /* revert flags set by cgroup core while adding @cfts */
-+ cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL);
-+ }
-+}
-+
-+static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
-+{
-+ struct cftype *cft;
-+
-+ for (cft = cfts; cft->name[0] != '\0'; cft++) {
-+ struct kernfs_ops *kf_ops;
-+
-+ WARN_ON(cft->ss || cft->kf_ops);
-+
-+ if (cft->seq_start)
-+ kf_ops = &cgroup_kf_ops;
-+ else
-+ kf_ops = &cgroup_kf_single_ops;
-+
-+ /*
-+ * Ugh... if @cft wants a custom max_write_len, we need to
-+ * make a copy of kf_ops to set its atomic_write_len.
-+ */
-+ if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
-+ kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
-+ if (!kf_ops) {
-+ cgroup_exit_cftypes(cfts);
-+ return -ENOMEM;
-+ }
-+ kf_ops->atomic_write_len = cft->max_write_len;
-+ }
-+
-+ cft->kf_ops = kf_ops;
-+ cft->ss = ss;
-+ }
-+
-+ return 0;
-+}
-+
-+static int cgroup_rm_cftypes_locked(struct cftype *cfts)
-+{
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ if (!cfts || !cfts[0].ss)
-+ return -ENOENT;
-+
-+ list_del(&cfts->node);
-+ cgroup_apply_cftypes(cfts, false);
-+ cgroup_exit_cftypes(cfts);
-+ return 0;
-+}
-+
-+/**
-+ * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
-+ * @cfts: zero-length name terminated array of cftypes
-+ *
-+ * Unregister @cfts. Files described by @cfts are removed from all
-+ * existing cgroups and all future cgroups won't have them either. This
-+ * function can be called anytime whether @cfts' subsys is attached or not.
-+ *
-+ * Returns 0 on successful unregistration, -ENOENT if @cfts is not
-+ * registered.
-+ */
-+int cgroup_rm_cftypes(struct cftype *cfts)
-+{
-+ int ret;
-+
-+ mutex_lock(&cgroup_mutex);
-+ ret = cgroup_rm_cftypes_locked(cfts);
-+ mutex_unlock(&cgroup_mutex);
-+ return ret;
-+}
-+
-+/**
-+ * cgroup_add_cftypes - add an array of cftypes to a subsystem
-+ * @ss: target cgroup subsystem
-+ * @cfts: zero-length name terminated array of cftypes
-+ *
-+ * Register @cfts to @ss. Files described by @cfts are created for all
-+ * existing cgroups to which @ss is attached and all future cgroups will
-+ * have them too. This function can be called anytime whether @ss is
-+ * attached or not.
-+ *
-+ * Returns 0 on successful registration, -errno on failure. Note that this
-+ * function currently returns 0 as long as @cfts registration is successful
-+ * even if some file creation attempts on existing cgroups fail.
-+ */
-+static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
-+{
-+ int ret;
-+
-+ if (ss->disabled)
-+ return 0;
-+
-+ if (!cfts || cfts[0].name[0] == '\0')
-+ return 0;
-+
-+ ret = cgroup_init_cftypes(ss, cfts);
-+ if (ret)
-+ return ret;
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ list_add_tail(&cfts->node, &ss->cfts);
-+ ret = cgroup_apply_cftypes(cfts, true);
-+ if (ret)
-+ cgroup_rm_cftypes_locked(cfts);
-+
-+ mutex_unlock(&cgroup_mutex);
-+ return ret;
-+}
-+
-+/**
-+ * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
-+ * @ss: target cgroup subsystem
-+ * @cfts: zero-length name terminated array of cftypes
-+ *
-+ * Similar to cgroup_add_cftypes() but the added files are only used for
-+ * the default hierarchy.
-+ */
-+int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
-+{
-+ struct cftype *cft;
-+
-+ for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
-+ cft->flags |= __CFTYPE_ONLY_ON_DFL;
-+ return cgroup_add_cftypes(ss, cfts);
-+}
-+
-+/**
-+ * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
-+ * @ss: target cgroup subsystem
-+ * @cfts: zero-length name terminated array of cftypes
-+ *
-+ * Similar to cgroup_add_cftypes() but the added files are only used for
-+ * the legacy hierarchies.
-+ */
-+int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
-+{
-+ struct cftype *cft;
-+
-+ /*
-+ * If legacy_flies_on_dfl, we want to show the legacy files on the
-+ * dfl hierarchy but iff the target subsystem hasn't been updated
-+ * for the dfl hierarchy yet.
-+ */
-+ if (!cgroup_legacy_files_on_dfl ||
-+ ss->dfl_cftypes != ss->legacy_cftypes) {
-+ for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
-+ cft->flags |= __CFTYPE_NOT_ON_DFL;
-+ }
-+
-+ return cgroup_add_cftypes(ss, cfts);
-+}
-+
-+/**
-+ * cgroup_task_count - count the number of tasks in a cgroup.
-+ * @cgrp: the cgroup in question
-+ *
-+ * Return the number of tasks in the cgroup.
-+ */
-+static int cgroup_task_count(const struct cgroup *cgrp)
-+{
-+ int count = 0;
-+ struct cgrp_cset_link *link;
-+
-+ down_read(&css_set_rwsem);
-+ list_for_each_entry(link, &cgrp->cset_links, cset_link)
-+ count += atomic_read(&link->cset->refcount);
-+ up_read(&css_set_rwsem);
-+ return count;
-+}
-+
-+/**
-+ * css_next_child - find the next child of a given css
-+ * @pos: the current position (%NULL to initiate traversal)
-+ * @parent: css whose children to walk
-+ *
-+ * This function returns the next child of @parent and should be called
-+ * under either cgroup_mutex or RCU read lock. The only requirement is
-+ * that @parent and @pos are accessible. The next sibling is guaranteed to
-+ * be returned regardless of their states.
-+ *
-+ * If a subsystem synchronizes ->css_online() and the start of iteration, a
-+ * css which finished ->css_online() is guaranteed to be visible in the
-+ * future iterations and will stay visible until the last reference is put.
-+ * A css which hasn't finished ->css_online() or already finished
-+ * ->css_offline() may show up during traversal. It's each subsystem's
-+ * responsibility to synchronize against on/offlining.
-+ */
-+struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
-+ struct cgroup_subsys_state *parent)
-+{
-+ struct cgroup_subsys_state *next;
-+
-+ cgroup_assert_mutex_or_rcu_locked();
-+
-+ /*
-+ * @pos could already have been unlinked from the sibling list.
-+ * Once a cgroup is removed, its ->sibling.next is no longer
-+ * updated when its next sibling changes. CSS_RELEASED is set when
-+ * @pos is taken off list, at which time its next pointer is valid,
-+ * and, as releases are serialized, the one pointed to by the next
-+ * pointer is guaranteed to not have started release yet. This
-+ * implies that if we observe !CSS_RELEASED on @pos in this RCU
-+ * critical section, the one pointed to by its next pointer is
-+ * guaranteed to not have finished its RCU grace period even if we
-+ * have dropped rcu_read_lock() inbetween iterations.
-+ *
-+ * If @pos has CSS_RELEASED set, its next pointer can't be
-+ * dereferenced; however, as each css is given a monotonically
-+ * increasing unique serial number and always appended to the
-+ * sibling list, the next one can be found by walking the parent's
-+ * children until the first css with higher serial number than
-+ * @pos's. While this path can be slower, it happens iff iteration
-+ * races against release and the race window is very small.
-+ */
-+ if (!pos) {
-+ next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
-+ } else if (likely(!(pos->flags & CSS_RELEASED))) {
-+ next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
-+ } else {
-+ list_for_each_entry_rcu(next, &parent->children, sibling)
-+ if (next->serial_nr > pos->serial_nr)
-+ break;
-+ }
-+
-+ /*
-+ * @next, if not pointing to the head, can be dereferenced and is
-+ * the next sibling.
-+ */
-+ if (&next->sibling != &parent->children)
-+ return next;
-+ return NULL;
-+}
-+
-+/**
-+ * css_next_descendant_pre - find the next descendant for pre-order walk
-+ * @pos: the current position (%NULL to initiate traversal)
-+ * @root: css whose descendants to walk
-+ *
-+ * To be used by css_for_each_descendant_pre(). Find the next descendant
-+ * to visit for pre-order traversal of @root's descendants. @root is
-+ * included in the iteration and the first node to be visited.
-+ *
-+ * While this function requires cgroup_mutex or RCU read locking, it
-+ * doesn't require the whole traversal to be contained in a single critical
-+ * section. This function will return the correct next descendant as long
-+ * as both @pos and @root are accessible and @pos is a descendant of @root.
-+ *
-+ * If a subsystem synchronizes ->css_online() and the start of iteration, a
-+ * css which finished ->css_online() is guaranteed to be visible in the
-+ * future iterations and will stay visible until the last reference is put.
-+ * A css which hasn't finished ->css_online() or already finished
-+ * ->css_offline() may show up during traversal. It's each subsystem's
-+ * responsibility to synchronize against on/offlining.
-+ */
-+struct cgroup_subsys_state *
-+css_next_descendant_pre(struct cgroup_subsys_state *pos,
-+ struct cgroup_subsys_state *root)
-+{
-+ struct cgroup_subsys_state *next;
-+
-+ cgroup_assert_mutex_or_rcu_locked();
-+
-+ /* if first iteration, visit @root */
-+ if (!pos)
-+ return root;
-+
-+ /* visit the first child if exists */
-+ next = css_next_child(NULL, pos);
-+ if (next)
-+ return next;
-+
-+ /* no child, visit my or the closest ancestor's next sibling */
-+ while (pos != root) {
-+ next = css_next_child(pos, pos->parent);
-+ if (next)
-+ return next;
-+ pos = pos->parent;
-+ }
-+
-+ return NULL;
-+}
-+
-+/**
-+ * css_rightmost_descendant - return the rightmost descendant of a css
-+ * @pos: css of interest
-+ *
-+ * Return the rightmost descendant of @pos. If there's no descendant, @pos
-+ * is returned. This can be used during pre-order traversal to skip
-+ * subtree of @pos.
-+ *
-+ * While this function requires cgroup_mutex or RCU read locking, it
-+ * doesn't require the whole traversal to be contained in a single critical
-+ * section. This function will return the correct rightmost descendant as
-+ * long as @pos is accessible.
-+ */
-+struct cgroup_subsys_state *
-+css_rightmost_descendant(struct cgroup_subsys_state *pos)
-+{
-+ struct cgroup_subsys_state *last, *tmp;
-+
-+ cgroup_assert_mutex_or_rcu_locked();
-+
-+ do {
-+ last = pos;
-+ /* ->prev isn't RCU safe, walk ->next till the end */
-+ pos = NULL;
-+ css_for_each_child(tmp, last)
-+ pos = tmp;
-+ } while (pos);
-+
-+ return last;
-+}
-+
-+static struct cgroup_subsys_state *
-+css_leftmost_descendant(struct cgroup_subsys_state *pos)
-+{
-+ struct cgroup_subsys_state *last;
-+
-+ do {
-+ last = pos;
-+ pos = css_next_child(NULL, pos);
-+ } while (pos);
-+
-+ return last;
-+}
-+
-+/**
-+ * css_next_descendant_post - find the next descendant for post-order walk
-+ * @pos: the current position (%NULL to initiate traversal)
-+ * @root: css whose descendants to walk
-+ *
-+ * To be used by css_for_each_descendant_post(). Find the next descendant
-+ * to visit for post-order traversal of @root's descendants. @root is
-+ * included in the iteration and the last node to be visited.
-+ *
-+ * While this function requires cgroup_mutex or RCU read locking, it
-+ * doesn't require the whole traversal to be contained in a single critical
-+ * section. This function will return the correct next descendant as long
-+ * as both @pos and @cgroup are accessible and @pos is a descendant of
-+ * @cgroup.
-+ *
-+ * If a subsystem synchronizes ->css_online() and the start of iteration, a
-+ * css which finished ->css_online() is guaranteed to be visible in the
-+ * future iterations and will stay visible until the last reference is put.
-+ * A css which hasn't finished ->css_online() or already finished
-+ * ->css_offline() may show up during traversal. It's each subsystem's
-+ * responsibility to synchronize against on/offlining.
-+ */
-+struct cgroup_subsys_state *
-+css_next_descendant_post(struct cgroup_subsys_state *pos,
-+ struct cgroup_subsys_state *root)
-+{
-+ struct cgroup_subsys_state *next;
-+
-+ cgroup_assert_mutex_or_rcu_locked();
-+
-+ /* if first iteration, visit leftmost descendant which may be @root */
-+ if (!pos)
-+ return css_leftmost_descendant(root);
-+
-+ /* if we visited @root, we're done */
-+ if (pos == root)
-+ return NULL;
-+
-+ /* if there's an unvisited sibling, visit its leftmost descendant */
-+ next = css_next_child(pos, pos->parent);
-+ if (next)
-+ return css_leftmost_descendant(next);
-+
-+ /* no sibling left, visit parent */
-+ return pos->parent;
-+}
-+
-+/**
-+ * css_has_online_children - does a css have online children
-+ * @css: the target css
-+ *
-+ * Returns %true if @css has any online children; otherwise, %false. This
-+ * function can be called from any context but the caller is responsible
-+ * for synchronizing against on/offlining as necessary.
-+ */
-+bool css_has_online_children(struct cgroup_subsys_state *css)
-+{
-+ struct cgroup_subsys_state *child;
-+ bool ret = false;
-+
-+ rcu_read_lock();
-+ css_for_each_child(child, css) {
-+ if (child->flags & CSS_ONLINE) {
-+ ret = true;
-+ break;
-+ }
-+ }
-+ rcu_read_unlock();
-+ return ret;
-+}
-+
-+/**
-+ * css_advance_task_iter - advance a task itererator to the next css_set
-+ * @it: the iterator to advance
-+ *
-+ * Advance @it to the next css_set to walk.
-+ */
-+static void css_advance_task_iter(struct css_task_iter *it)
-+{
-+ struct list_head *l = it->cset_pos;
-+ struct cgrp_cset_link *link;
-+ struct css_set *cset;
-+
-+ /* Advance to the next non-empty css_set */
-+ do {
-+ l = l->next;
-+ if (l == it->cset_head) {
-+ it->cset_pos = NULL;
-+ return;
-+ }
-+
-+ if (it->ss) {
-+ cset = container_of(l, struct css_set,
-+ e_cset_node[it->ss->id]);
-+ } else {
-+ link = list_entry(l, struct cgrp_cset_link, cset_link);
-+ cset = link->cset;
-+ }
-+ } while (list_empty(&cset->tasks) && list_empty(&cset->mg_tasks));
-+
-+ it->cset_pos = l;
-+
-+ if (!list_empty(&cset->tasks))
-+ it->task_pos = cset->tasks.next;
-+ else
-+ it->task_pos = cset->mg_tasks.next;
-+
-+ it->tasks_head = &cset->tasks;
-+ it->mg_tasks_head = &cset->mg_tasks;
-+}
-+
-+/**
-+ * css_task_iter_start - initiate task iteration
-+ * @css: the css to walk tasks of
-+ * @it: the task iterator to use
-+ *
-+ * Initiate iteration through the tasks of @css. The caller can call
-+ * css_task_iter_next() to walk through the tasks until the function
-+ * returns NULL. On completion of iteration, css_task_iter_end() must be
-+ * called.
-+ *
-+ * Note that this function acquires a lock which is released when the
-+ * iteration finishes. The caller can't sleep while iteration is in
-+ * progress.
-+ */
-+void css_task_iter_start(struct cgroup_subsys_state *css,
-+ struct css_task_iter *it)
-+ __acquires(css_set_rwsem)
-+{
-+ /* no one should try to iterate before mounting cgroups */
-+ WARN_ON_ONCE(!use_task_css_set_links);
-+
-+ down_read(&css_set_rwsem);
-+
-+ it->ss = css->ss;
-+
-+ if (it->ss)
-+ it->cset_pos = &css->cgroup->e_csets[css->ss->id];
-+ else
-+ it->cset_pos = &css->cgroup->cset_links;
-+
-+ it->cset_head = it->cset_pos;
-+
-+ css_advance_task_iter(it);
-+}
-+
-+/**
-+ * css_task_iter_next - return the next task for the iterator
-+ * @it: the task iterator being iterated
-+ *
-+ * The "next" function for task iteration. @it should have been
-+ * initialized via css_task_iter_start(). Returns NULL when the iteration
-+ * reaches the end.
-+ */
-+struct task_struct *css_task_iter_next(struct css_task_iter *it)
-+{
-+ struct task_struct *res;
-+ struct list_head *l = it->task_pos;
-+
-+ /* If the iterator cg is NULL, we have no tasks */
-+ if (!it->cset_pos)
-+ return NULL;
-+ res = list_entry(l, struct task_struct, cg_list);
-+
-+ /*
-+ * Advance iterator to find next entry. cset->tasks is consumed
-+ * first and then ->mg_tasks. After ->mg_tasks, we move onto the
-+ * next cset.
-+ */
-+ l = l->next;
-+
-+ if (l == it->tasks_head)
-+ l = it->mg_tasks_head->next;
-+
-+ if (l == it->mg_tasks_head)
-+ css_advance_task_iter(it);
-+ else
-+ it->task_pos = l;
-+
-+ return res;
-+}
-+
-+/**
-+ * css_task_iter_end - finish task iteration
-+ * @it: the task iterator to finish
-+ *
-+ * Finish task iteration started by css_task_iter_start().
-+ */
-+void css_task_iter_end(struct css_task_iter *it)
-+ __releases(css_set_rwsem)
-+{
-+ up_read(&css_set_rwsem);
-+}
-+
-+/**
-+ * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
-+ * @to: cgroup to which the tasks will be moved
-+ * @from: cgroup in which the tasks currently reside
-+ *
-+ * Locking rules between cgroup_post_fork() and the migration path
-+ * guarantee that, if a task is forking while being migrated, the new child
-+ * is guaranteed to be either visible in the source cgroup after the
-+ * parent's migration is complete or put into the target cgroup. No task
-+ * can slip out of migration through forking.
-+ */
-+int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
-+{
-+ LIST_HEAD(preloaded_csets);
-+ struct cgrp_cset_link *link;
-+ struct css_task_iter it;
-+ struct task_struct *task;
-+ int ret;
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ /* all tasks in @from are being moved, all csets are source */
-+ down_read(&css_set_rwsem);
-+ list_for_each_entry(link, &from->cset_links, cset_link)
-+ cgroup_migrate_add_src(link->cset, to, &preloaded_csets);
-+ up_read(&css_set_rwsem);
-+
-+ ret = cgroup_migrate_prepare_dst(to, &preloaded_csets);
-+ if (ret)
-+ goto out_err;
-+
-+ /*
-+ * Migrate tasks one-by-one until @form is empty. This fails iff
-+ * ->can_attach() fails.
-+ */
-+ do {
-+ css_task_iter_start(&from->self, &it);
-+ task = css_task_iter_next(&it);
-+ if (task)
-+ get_task_struct(task);
-+ css_task_iter_end(&it);
-+
-+ if (task) {
-+ ret = cgroup_migrate(to, task, false);
-+ put_task_struct(task);
-+ }
-+ } while (task && !ret);
-+out_err:
-+ cgroup_migrate_finish(&preloaded_csets);
-+ mutex_unlock(&cgroup_mutex);
-+ return ret;
-+}
-+
-+/*
-+ * Stuff for reading the 'tasks'/'procs' files.
-+ *
-+ * Reading this file can return large amounts of data if a cgroup has
-+ * *lots* of attached tasks. So it may need several calls to read(),
-+ * but we cannot guarantee that the information we produce is correct
-+ * unless we produce it entirely atomically.
-+ *
-+ */
-+
-+/* which pidlist file are we talking about? */
-+enum cgroup_filetype {
-+ CGROUP_FILE_PROCS,
-+ CGROUP_FILE_TASKS,
-+};
-+
-+/*
-+ * A pidlist is a list of pids that virtually represents the contents of one
-+ * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
-+ * a pair (one each for procs, tasks) for each pid namespace that's relevant
-+ * to the cgroup.
-+ */
-+struct cgroup_pidlist {
-+ /*
-+ * used to find which pidlist is wanted. doesn't change as long as
-+ * this particular list stays in the list.
-+ */
-+ struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
-+ /* array of xids */
-+ pid_t *list;
-+ /* how many elements the above list has */
-+ int length;
-+ /* each of these stored in a list by its cgroup */
-+ struct list_head links;
-+ /* pointer to the cgroup we belong to, for list removal purposes */
-+ struct cgroup *owner;
-+ /* for delayed destruction */
-+ struct delayed_work destroy_dwork;
-+};
-+
-+/*
-+ * The following two functions "fix" the issue where there are more pids
-+ * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
-+ * TODO: replace with a kernel-wide solution to this problem
-+ */
-+#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
-+static void *pidlist_allocate(int count)
-+{
-+ if (PIDLIST_TOO_LARGE(count))
-+ return vmalloc(count * sizeof(pid_t));
-+ else
-+ return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
-+}
-+
-+static void pidlist_free(void *p)
-+{
-+ kvfree(p);
-+}
-+
-+/*
-+ * Used to destroy all pidlists lingering waiting for destroy timer. None
-+ * should be left afterwards.
-+ */
-+static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
-+{
-+ struct cgroup_pidlist *l, *tmp_l;
-+
-+ mutex_lock(&cgrp->pidlist_mutex);
-+ list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
-+ mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
-+ mutex_unlock(&cgrp->pidlist_mutex);
-+
-+ flush_workqueue(cgroup_pidlist_destroy_wq);
-+ BUG_ON(!list_empty(&cgrp->pidlists));
-+}
-+
-+static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
-+{
-+ struct delayed_work *dwork = to_delayed_work(work);
-+ struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
-+ destroy_dwork);
-+ struct cgroup_pidlist *tofree = NULL;
-+
-+ mutex_lock(&l->owner->pidlist_mutex);
-+
-+ /*
-+ * Destroy iff we didn't get queued again. The state won't change
-+ * as destroy_dwork can only be queued while locked.
-+ */
-+ if (!delayed_work_pending(dwork)) {
-+ list_del(&l->links);
-+ pidlist_free(l->list);
-+ put_pid_ns(l->key.ns);
-+ tofree = l;
-+ }
-+
-+ mutex_unlock(&l->owner->pidlist_mutex);
-+ kfree(tofree);
-+}
-+
-+/*
-+ * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
-+ * Returns the number of unique elements.
-+ */
-+static int pidlist_uniq(pid_t *list, int length)
-+{
-+ int src, dest = 1;
-+
-+ /*
-+ * we presume the 0th element is unique, so i starts at 1. trivial
-+ * edge cases first; no work needs to be done for either
-+ */
-+ if (length == 0 || length == 1)
-+ return length;
-+ /* src and dest walk down the list; dest counts unique elements */
-+ for (src = 1; src < length; src++) {
-+ /* find next unique element */
-+ while (list[src] == list[src-1]) {
-+ src++;
-+ if (src == length)
-+ goto after;
-+ }
-+ /* dest always points to where the next unique element goes */
-+ list[dest] = list[src];
-+ dest++;
-+ }
-+after:
-+ return dest;
-+}
-+
-+/*
-+ * The two pid files - task and cgroup.procs - guaranteed that the result
-+ * is sorted, which forced this whole pidlist fiasco. As pid order is
-+ * different per namespace, each namespace needs differently sorted list,
-+ * making it impossible to use, for example, single rbtree of member tasks
-+ * sorted by task pointer. As pidlists can be fairly large, allocating one
-+ * per open file is dangerous, so cgroup had to implement shared pool of
-+ * pidlists keyed by cgroup and namespace.
-+ *
-+ * All this extra complexity was caused by the original implementation
-+ * committing to an entirely unnecessary property. In the long term, we
-+ * want to do away with it. Explicitly scramble sort order if on the
-+ * default hierarchy so that no such expectation exists in the new
-+ * interface.
-+ *
-+ * Scrambling is done by swapping every two consecutive bits, which is
-+ * non-identity one-to-one mapping which disturbs sort order sufficiently.
-+ */
-+static pid_t pid_fry(pid_t pid)
-+{
-+ unsigned a = pid & 0x55555555;
-+ unsigned b = pid & 0xAAAAAAAA;
-+
-+ return (a << 1) | (b >> 1);
-+}
-+
-+static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
-+{
-+ if (cgroup_on_dfl(cgrp))
-+ return pid_fry(pid);
-+ else
-+ return pid;
-+}
-+
-+static int cmppid(const void *a, const void *b)
-+{
-+ return *(pid_t *)a - *(pid_t *)b;
-+}
-+
-+static int fried_cmppid(const void *a, const void *b)
-+{
-+ return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b);
-+}
-+
-+static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
-+ enum cgroup_filetype type)
-+{
-+ struct cgroup_pidlist *l;
-+ /* don't need task_nsproxy() if we're looking at ourself */
-+ struct pid_namespace *ns = task_active_pid_ns(current);
-+
-+ lockdep_assert_held(&cgrp->pidlist_mutex);
-+
-+ list_for_each_entry(l, &cgrp->pidlists, links)
-+ if (l->key.type == type && l->key.ns == ns)
-+ return l;
-+ return NULL;
-+}
-+
-+/*
-+ * find the appropriate pidlist for our purpose (given procs vs tasks)
-+ * returns with the lock on that pidlist already held, and takes care
-+ * of the use count, or returns NULL with no locks held if we're out of
-+ * memory.
-+ */
-+static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
-+ enum cgroup_filetype type)
-+{
-+ struct cgroup_pidlist *l;
-+
-+ lockdep_assert_held(&cgrp->pidlist_mutex);
-+
-+ l = cgroup_pidlist_find(cgrp, type);
-+ if (l)
-+ return l;
-+
-+ /* entry not found; create a new one */
-+ l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
-+ if (!l)
-+ return l;
-+
-+ INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
-+ l->key.type = type;
-+ /* don't need task_nsproxy() if we're looking at ourself */
-+ l->key.ns = get_pid_ns(task_active_pid_ns(current));
-+ l->owner = cgrp;
-+ list_add(&l->links, &cgrp->pidlists);
-+ return l;
-+}
-+
-+/*
-+ * Load a cgroup's pidarray with either procs' tgids or tasks' pids
-+ */
-+static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
-+ struct cgroup_pidlist **lp)
-+{
-+ pid_t *array;
-+ int length;
-+ int pid, n = 0; /* used for populating the array */
-+ struct css_task_iter it;
-+ struct task_struct *tsk;
-+ struct cgroup_pidlist *l;
-+
-+ lockdep_assert_held(&cgrp->pidlist_mutex);
-+
-+ /*
-+ * If cgroup gets more users after we read count, we won't have
-+ * enough space - tough. This race is indistinguishable to the
-+ * caller from the case that the additional cgroup users didn't
-+ * show up until sometime later on.
-+ */
-+ length = cgroup_task_count(cgrp);
-+ array = pidlist_allocate(length);
-+ if (!array)
-+ return -ENOMEM;
-+ /* now, populate the array */
-+ css_task_iter_start(&cgrp->self, &it);
-+ while ((tsk = css_task_iter_next(&it))) {
-+ if (unlikely(n == length))
-+ break;
-+ /* get tgid or pid for procs or tasks file respectively */
-+ if (type == CGROUP_FILE_PROCS)
-+ pid = task_tgid_vnr(tsk);
-+ else
-+ pid = task_pid_vnr(tsk);
-+ if (pid > 0) /* make sure to only use valid results */
-+ array[n++] = pid;
-+ }
-+ css_task_iter_end(&it);
-+ length = n;
-+ /* now sort & (if procs) strip out duplicates */
-+ if (cgroup_on_dfl(cgrp))
-+ sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
-+ else
-+ sort(array, length, sizeof(pid_t), cmppid, NULL);
-+ if (type == CGROUP_FILE_PROCS)
-+ length = pidlist_uniq(array, length);
-+
-+ l = cgroup_pidlist_find_create(cgrp, type);
-+ if (!l) {
-+ pidlist_free(array);
-+ return -ENOMEM;
-+ }
-+
-+ /* store array, freeing old if necessary */
-+ pidlist_free(l->list);
-+ l->list = array;
-+ l->length = length;
-+ *lp = l;
-+ return 0;
-+}
-+
-+/**
-+ * cgroupstats_build - build and fill cgroupstats
-+ * @stats: cgroupstats to fill information into
-+ * @dentry: A dentry entry belonging to the cgroup for which stats have
-+ * been requested.
-+ *
-+ * Build and fill cgroupstats so that taskstats can export it to user
-+ * space.
-+ */
-+int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
-+{
-+ struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
-+ struct cgroup *cgrp;
-+ struct css_task_iter it;
-+ struct task_struct *tsk;
-+
-+ /* it should be kernfs_node belonging to cgroupfs and is a directory */
-+ if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
-+ kernfs_type(kn) != KERNFS_DIR)
-+ return -EINVAL;
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ /*
-+ * We aren't being called from kernfs and there's no guarantee on
-+ * @kn->priv's validity. For this and css_tryget_online_from_dir(),
-+ * @kn->priv is RCU safe. Let's do the RCU dancing.
-+ */
-+ rcu_read_lock();
-+ cgrp = rcu_dereference(kn->priv);
-+ if (!cgrp || cgroup_is_dead(cgrp)) {
-+ rcu_read_unlock();
-+ mutex_unlock(&cgroup_mutex);
-+ return -ENOENT;
-+ }
-+ rcu_read_unlock();
-+
-+ css_task_iter_start(&cgrp->self, &it);
-+ while ((tsk = css_task_iter_next(&it))) {
-+ switch (tsk->state) {
-+ case TASK_RUNNING:
-+ stats->nr_running++;
-+ break;
-+ case TASK_INTERRUPTIBLE:
-+ stats->nr_sleeping++;
-+ break;
-+ case TASK_UNINTERRUPTIBLE:
-+ stats->nr_uninterruptible++;
-+ break;
-+ case TASK_STOPPED:
-+ stats->nr_stopped++;
-+ break;
-+ default:
-+ if (delayacct_is_task_waiting_on_io(tsk))
-+ stats->nr_io_wait++;
-+ break;
-+ }
-+ }
-+ css_task_iter_end(&it);
-+
-+ mutex_unlock(&cgroup_mutex);
-+ return 0;
-+}
-+
-+
-+/*
-+ * seq_file methods for the tasks/procs files. The seq_file position is the
-+ * next pid to display; the seq_file iterator is a pointer to the pid
-+ * in the cgroup->l->list array.
-+ */
-+
-+static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
-+{
-+ /*
-+ * Initially we receive a position value that corresponds to
-+ * one more than the last pid shown (or 0 on the first call or
-+ * after a seek to the start). Use a binary-search to find the
-+ * next pid to display, if any
-+ */
-+ struct kernfs_open_file *of = s->private;
-+ struct cgroup *cgrp = seq_css(s)->cgroup;
-+ struct cgroup_pidlist *l;
-+ enum cgroup_filetype type = seq_cft(s)->private;
-+ int index = 0, pid = *pos;
-+ int *iter, ret;
-+
-+ mutex_lock(&cgrp->pidlist_mutex);
-+
-+ /*
-+ * !NULL @of->priv indicates that this isn't the first start()
-+ * after open. If the matching pidlist is around, we can use that.
-+ * Look for it. Note that @of->priv can't be used directly. It
-+ * could already have been destroyed.
-+ */
-+ if (of->priv)
-+ of->priv = cgroup_pidlist_find(cgrp, type);
-+
-+ /*
-+ * Either this is the first start() after open or the matching
-+ * pidlist has been destroyed inbetween. Create a new one.
-+ */
-+ if (!of->priv) {
-+ ret = pidlist_array_load(cgrp, type,
-+ (struct cgroup_pidlist **)&of->priv);
-+ if (ret)
-+ return ERR_PTR(ret);
-+ }
-+ l = of->priv;
-+
-+ if (pid) {
-+ int end = l->length;
-+
-+ while (index < end) {
-+ int mid = (index + end) / 2;
-+ if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
-+ index = mid;
-+ break;
-+ } else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
-+ index = mid + 1;
-+ else
-+ end = mid;
-+ }
-+ }
-+ /* If we're off the end of the array, we're done */
-+ if (index >= l->length)
-+ return NULL;
-+ /* Update the abstract position to be the actual pid that we found */
-+ iter = l->list + index;
-+ *pos = cgroup_pid_fry(cgrp, *iter);
-+ return iter;
-+}
-+
-+static void cgroup_pidlist_stop(struct seq_file *s, void *v)
-+{
-+ struct kernfs_open_file *of = s->private;
-+ struct cgroup_pidlist *l = of->priv;
-+
-+ if (l)
-+ mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
-+ CGROUP_PIDLIST_DESTROY_DELAY);
-+ mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
-+}
-+
-+static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
-+{
-+ struct kernfs_open_file *of = s->private;
-+ struct cgroup_pidlist *l = of->priv;
-+ pid_t *p = v;
-+ pid_t *end = l->list + l->length;
-+ /*
-+ * Advance to the next pid in the array. If this goes off the
-+ * end, we're done
-+ */
-+ p++;
-+ if (p >= end) {
-+ return NULL;
-+ } else {
-+ *pos = cgroup_pid_fry(seq_css(s)->cgroup, *p);
-+ return p;
-+ }
-+}
-+
-+static int cgroup_pidlist_show(struct seq_file *s, void *v)
-+{
-+ seq_printf(s, "%d\n", *(int *)v);
-+
-+ return 0;
-+}
-+
-+static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ return notify_on_release(css->cgroup);
-+}
-+
-+static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
-+ struct cftype *cft, u64 val)
-+{
-+ if (val)
-+ set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
-+ else
-+ clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
-+ return 0;
-+}
-+
-+static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
-+}
-+
-+static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
-+ struct cftype *cft, u64 val)
-+{
-+ if (val)
-+ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
-+ else
-+ clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
-+ return 0;
-+}
-+
-+/* cgroup core interface files for the default hierarchy */
-+static struct cftype cgroup_dfl_base_files[] = {
-+ {
-+ .name = "cgroup.procs",
-+ .seq_start = cgroup_pidlist_start,
-+ .seq_next = cgroup_pidlist_next,
-+ .seq_stop = cgroup_pidlist_stop,
-+ .seq_show = cgroup_pidlist_show,
-+ .private = CGROUP_FILE_PROCS,
-+ .write = cgroup_procs_write,
-+ .mode = S_IRUGO | S_IWUSR,
-+ },
-+ {
-+ .name = "cgroup.controllers",
-+ .flags = CFTYPE_ONLY_ON_ROOT,
-+ .seq_show = cgroup_root_controllers_show,
-+ },
-+ {
-+ .name = "cgroup.controllers",
-+ .flags = CFTYPE_NOT_ON_ROOT,
-+ .seq_show = cgroup_controllers_show,
-+ },
-+ {
-+ .name = "cgroup.subtree_control",
-+ .seq_show = cgroup_subtree_control_show,
-+ .write = cgroup_subtree_control_write,
-+ },
-+ {
-+ .name = "cgroup.populated",
-+ .flags = CFTYPE_NOT_ON_ROOT,
-+ .seq_show = cgroup_populated_show,
-+ },
-+ { } /* terminate */
-+};
-+
-+/* cgroup core interface files for the legacy hierarchies */
-+static struct cftype cgroup_legacy_base_files[] = {
-+ {
-+ .name = "cgroup.procs",
-+ .seq_start = cgroup_pidlist_start,
-+ .seq_next = cgroup_pidlist_next,
-+ .seq_stop = cgroup_pidlist_stop,
-+ .seq_show = cgroup_pidlist_show,
-+ .private = CGROUP_FILE_PROCS,
-+ .write = cgroup_procs_write,
-+ .mode = S_IRUGO | S_IWUSR,
-+ },
-+ {
-+ .name = "cgroup.clone_children",
-+ .read_u64 = cgroup_clone_children_read,
-+ .write_u64 = cgroup_clone_children_write,
-+ },
-+ {
-+ .name = "cgroup.sane_behavior",
-+ .flags = CFTYPE_ONLY_ON_ROOT,
-+ .seq_show = cgroup_sane_behavior_show,
-+ },
-+ {
-+ .name = "tasks",
-+ .seq_start = cgroup_pidlist_start,
-+ .seq_next = cgroup_pidlist_next,
-+ .seq_stop = cgroup_pidlist_stop,
-+ .seq_show = cgroup_pidlist_show,
-+ .private = CGROUP_FILE_TASKS,
-+ .write = cgroup_tasks_write,
-+ .mode = S_IRUGO | S_IWUSR,
-+ },
-+ {
-+ .name = "notify_on_release",
-+ .read_u64 = cgroup_read_notify_on_release,
-+ .write_u64 = cgroup_write_notify_on_release,
-+ },
-+ {
-+ .name = "release_agent",
-+ .flags = CFTYPE_ONLY_ON_ROOT,
-+ .seq_show = cgroup_release_agent_show,
-+ .write = cgroup_release_agent_write,
-+ .max_write_len = PATH_MAX - 1,
-+ },
-+ { } /* terminate */
-+};
-+
-+/**
-+ * cgroup_populate_dir - create subsys files in a cgroup directory
-+ * @cgrp: target cgroup
-+ * @subsys_mask: mask of the subsystem ids whose files should be added
-+ *
-+ * On failure, no file is added.
-+ */
-+static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask)
-+{
-+ struct cgroup_subsys *ss;
-+ int i, ret = 0;
-+
-+ /* process cftsets of each subsystem */
-+ for_each_subsys(ss, i) {
-+ struct cftype *cfts;
-+
-+ if (!(subsys_mask & (1 << i)))
-+ continue;
-+
-+ list_for_each_entry(cfts, &ss->cfts, node) {
-+ ret = cgroup_addrm_files(cgrp, cfts, true);
-+ if (ret < 0)
-+ goto err;
-+ }
-+ }
-+ return 0;
-+err:
-+ cgroup_clear_dir(cgrp, subsys_mask);
-+ return ret;
-+}
-+
-+/*
-+ * css destruction is four-stage process.
-+ *
-+ * 1. Destruction starts. Killing of the percpu_ref is initiated.
-+ * Implemented in kill_css().
-+ *
-+ * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
-+ * and thus css_tryget_online() is guaranteed to fail, the css can be
-+ * offlined by invoking offline_css(). After offlining, the base ref is
-+ * put. Implemented in css_killed_work_fn().
-+ *
-+ * 3. When the percpu_ref reaches zero, the only possible remaining
-+ * accessors are inside RCU read sections. css_release() schedules the
-+ * RCU callback.
-+ *
-+ * 4. After the grace period, the css can be freed. Implemented in
-+ * css_free_work_fn().
-+ *
-+ * It is actually hairier because both step 2 and 4 require process context
-+ * and thus involve punting to css->destroy_work adding two additional
-+ * steps to the already complex sequence.
-+ */
-+static void css_free_work_fn(struct work_struct *work)
-+{
-+ struct cgroup_subsys_state *css =
-+ container_of(work, struct cgroup_subsys_state, destroy_work);
-+ struct cgroup_subsys *ss = css->ss;
-+ struct cgroup *cgrp = css->cgroup;
-+
-+ percpu_ref_exit(&css->refcnt);
-+
-+ if (ss) {
-+ /* css free path */
-+ int id = css->id;
-+
-+ if (css->parent)
-+ css_put(css->parent);
-+
-+ ss->css_free(css);
-+ cgroup_idr_remove(&ss->css_idr, id);
-+ cgroup_put(cgrp);
-+ } else {
-+ /* cgroup free path */
-+ atomic_dec(&cgrp->root->nr_cgrps);
-+ cgroup_pidlist_destroy_all(cgrp);
-+ cancel_work_sync(&cgrp->release_agent_work);
-+
-+ if (cgroup_parent(cgrp)) {
-+ /*
-+ * We get a ref to the parent, and put the ref when
-+ * this cgroup is being freed, so it's guaranteed
-+ * that the parent won't be destroyed before its
-+ * children.
-+ */
-+ cgroup_put(cgroup_parent(cgrp));
-+ kernfs_put(cgrp->kn);
-+ kfree(cgrp);
-+ } else {
-+ /*
-+ * This is root cgroup's refcnt reaching zero,
-+ * which indicates that the root should be
-+ * released.
-+ */
-+ cgroup_destroy_root(cgrp->root);
-+ }
-+ }
-+}
-+
-+static void css_free_rcu_fn(struct rcu_head *rcu_head)
-+{
-+ struct cgroup_subsys_state *css =
-+ container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
-+
-+ INIT_WORK(&css->destroy_work, css_free_work_fn);
-+ queue_work(cgroup_destroy_wq, &css->destroy_work);
-+}
-+
-+static void css_release_work_fn(struct work_struct *work)
-+{
-+ struct cgroup_subsys_state *css =
-+ container_of(work, struct cgroup_subsys_state, destroy_work);
-+ struct cgroup_subsys *ss = css->ss;
-+ struct cgroup *cgrp = css->cgroup;
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ css->flags |= CSS_RELEASED;
-+ list_del_rcu(&css->sibling);
-+
-+ if (ss) {
-+ /* css release path */
-+ cgroup_idr_replace(&ss->css_idr, NULL, css->id);
-+ if (ss->css_released)
-+ ss->css_released(css);
-+ } else {
-+ /* cgroup release path */
-+ cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
-+ cgrp->id = -1;
-+
-+ /*
-+ * There are two control paths which try to determine
-+ * cgroup from dentry without going through kernfs -
-+ * cgroupstats_build() and css_tryget_online_from_dir().
-+ * Those are supported by RCU protecting clearing of
-+ * cgrp->kn->priv backpointer.
-+ */
-+ RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, NULL);
-+ }
-+
-+ mutex_unlock(&cgroup_mutex);
-+
-+ call_rcu(&css->rcu_head, css_free_rcu_fn);
-+}
-+
-+static void css_release(struct percpu_ref *ref)
-+{
-+ struct cgroup_subsys_state *css =
-+ container_of(ref, struct cgroup_subsys_state, refcnt);
-+
-+ INIT_WORK(&css->destroy_work, css_release_work_fn);
-+ queue_work(cgroup_destroy_wq, &css->destroy_work);
-+}
-+
-+static void init_and_link_css(struct cgroup_subsys_state *css,
-+ struct cgroup_subsys *ss, struct cgroup *cgrp)
-+{
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ cgroup_get(cgrp);
-+
-+ memset(css, 0, sizeof(*css));
-+ css->cgroup = cgrp;
-+ css->ss = ss;
-+ INIT_LIST_HEAD(&css->sibling);
-+ INIT_LIST_HEAD(&css->children);
-+ css->serial_nr = css_serial_nr_next++;
-+
-+ if (cgroup_parent(cgrp)) {
-+ css->parent = cgroup_css(cgroup_parent(cgrp), ss);
-+ css_get(css->parent);
-+ }
-+
-+ BUG_ON(cgroup_css(cgrp, ss));
-+}
-+
-+/* invoke ->css_online() on a new CSS and mark it online if successful */
-+static int online_css(struct cgroup_subsys_state *css)
-+{
-+ struct cgroup_subsys *ss = css->ss;
-+ int ret = 0;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ if (ss->css_online)
-+ ret = ss->css_online(css);
-+ if (!ret) {
-+ css->flags |= CSS_ONLINE;
-+ rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
-+ }
-+ return ret;
-+}
-+
-+/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
-+static void offline_css(struct cgroup_subsys_state *css)
-+{
-+ struct cgroup_subsys *ss = css->ss;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ if (!(css->flags & CSS_ONLINE))
-+ return;
-+
-+ if (ss->css_offline)
-+ ss->css_offline(css);
-+
-+ css->flags &= ~CSS_ONLINE;
-+ RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
-+
-+ wake_up_all(&css->cgroup->offline_waitq);
-+}
-+
-+/**
-+ * create_css - create a cgroup_subsys_state
-+ * @cgrp: the cgroup new css will be associated with
-+ * @ss: the subsys of new css
-+ * @visible: whether to create control knobs for the new css or not
-+ *
-+ * Create a new css associated with @cgrp - @ss pair. On success, the new
-+ * css is online and installed in @cgrp with all interface files created if
-+ * @visible. Returns 0 on success, -errno on failure.
-+ */
-+static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
-+ bool visible)
-+{
-+ struct cgroup *parent = cgroup_parent(cgrp);
-+ struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
-+ struct cgroup_subsys_state *css;
-+ int err;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ css = ss->css_alloc(parent_css);
-+ if (IS_ERR(css))
-+ return PTR_ERR(css);
-+
-+ init_and_link_css(css, ss, cgrp);
-+
-+ err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);
-+ if (err)
-+ goto err_free_css;
-+
-+ err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_NOWAIT);
-+ if (err < 0)
-+ goto err_free_percpu_ref;
-+ css->id = err;
-+
-+ if (visible) {
-+ err = cgroup_populate_dir(cgrp, 1 << ss->id);
-+ if (err)
-+ goto err_free_id;
-+ }
-+
-+ /* @css is ready to be brought online now, make it visible */
-+ list_add_tail_rcu(&css->sibling, &parent_css->children);
-+ cgroup_idr_replace(&ss->css_idr, css, css->id);
-+
-+ err = online_css(css);
-+ if (err)
-+ goto err_list_del;
-+
-+ if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
-+ cgroup_parent(parent)) {
-+ pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
-+ current->comm, current->pid, ss->name);
-+ if (!strcmp(ss->name, "memory"))
-+ pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
-+ ss->warned_broken_hierarchy = true;
-+ }
-+
-+ return 0;
-+
-+err_list_del:
-+ list_del_rcu(&css->sibling);
-+ cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
-+err_free_id:
-+ cgroup_idr_remove(&ss->css_idr, css->id);
-+err_free_percpu_ref:
-+ percpu_ref_exit(&css->refcnt);
-+err_free_css:
-+ call_rcu(&css->rcu_head, css_free_rcu_fn);
-+ return err;
-+}
-+
-+static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
-+ umode_t mode)
-+{
-+ struct cgroup *parent, *cgrp;
-+ struct cgroup_root *root;
-+ struct cgroup_subsys *ss;
-+ struct kernfs_node *kn;
-+ struct cftype *base_files;
-+ int ssid, ret;
-+
-+ /* Do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable.
-+ */
-+ if (strchr(name, '\n'))
-+ return -EINVAL;
-+
-+ parent = cgroup_kn_lock_live(parent_kn);
-+ if (!parent)
-+ return -ENODEV;
-+ root = parent->root;
-+
-+ /* allocate the cgroup and its ID, 0 is reserved for the root */
-+ cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL);
-+ if (!cgrp) {
-+ ret = -ENOMEM;
-+ goto out_unlock;
-+ }
-+
-+ ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL);
-+ if (ret)
-+ goto out_free_cgrp;
-+
-+ /*
-+ * Temporarily set the pointer to NULL, so idr_find() won't return
-+ * a half-baked cgroup.
-+ */
-+ cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_NOWAIT);
-+ if (cgrp->id < 0) {
-+ ret = -ENOMEM;
-+ goto out_cancel_ref;
-+ }
-+
-+ init_cgroup_housekeeping(cgrp);
-+
-+ cgrp->self.parent = &parent->self;
-+ cgrp->root = root;
-+
-+ if (notify_on_release(parent))
-+ set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
-+
-+ if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
-+ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
-+
-+ /* create the directory */
-+ kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
-+ if (IS_ERR(kn)) {
-+ ret = PTR_ERR(kn);
-+ goto out_free_id;
-+ }
-+ cgrp->kn = kn;
-+
-+ /*
-+ * This extra ref will be put in cgroup_free_fn() and guarantees
-+ * that @cgrp->kn is always accessible.
-+ */
-+ kernfs_get(kn);
-+
-+ cgrp->self.serial_nr = css_serial_nr_next++;
-+
-+ /* allocation complete, commit to creation */
-+ list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
-+ atomic_inc(&root->nr_cgrps);
-+ cgroup_get(parent);
-+
-+ /*
-+ * @cgrp is now fully operational. If something fails after this
-+ * point, it'll be released via the normal destruction path.
-+ */
-+ cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
-+
-+ ret = cgroup_kn_set_ugid(kn);
-+ if (ret)
-+ goto out_destroy;
-+
-+ if (cgroup_on_dfl(cgrp))
-+ base_files = cgroup_dfl_base_files;
-+ else
-+ base_files = cgroup_legacy_base_files;
-+
-+ ret = cgroup_addrm_files(cgrp, base_files, true);
-+ if (ret)
-+ goto out_destroy;
-+
-+ /* let's create and online css's */
-+ for_each_subsys(ss, ssid) {
-+ if (parent->child_subsys_mask & (1 << ssid)) {
-+ ret = create_css(cgrp, ss,
-+ parent->subtree_control & (1 << ssid));
-+ if (ret)
-+ goto out_destroy;
-+ }
-+ }
-+
-+ /*
-+ * On the default hierarchy, a child doesn't automatically inherit
-+ * subtree_control from the parent. Each is configured manually.
-+ */
-+ if (!cgroup_on_dfl(cgrp)) {
-+ cgrp->subtree_control = parent->subtree_control;
-+ cgroup_refresh_child_subsys_mask(cgrp);
-+ }
-+
-+ kernfs_activate(kn);
-+
-+ ret = 0;
-+ goto out_unlock;
-+
-+out_free_id:
-+ cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
-+out_cancel_ref:
-+ percpu_ref_exit(&cgrp->self.refcnt);
-+out_free_cgrp:
-+ kfree(cgrp);
-+out_unlock:
-+ cgroup_kn_unlock(parent_kn);
-+ return ret;
-+
-+out_destroy:
-+ cgroup_destroy_locked(cgrp);
-+ goto out_unlock;
-+}
-+
-+/*
-+ * This is called when the refcnt of a css is confirmed to be killed.
-+ * css_tryget_online() is now guaranteed to fail. Tell the subsystem to
-+ * initate destruction and put the css ref from kill_css().
-+ */
-+static void css_killed_work_fn(struct work_struct *work)
-+{
-+ struct cgroup_subsys_state *css =
-+ container_of(work, struct cgroup_subsys_state, destroy_work);
-+
-+ mutex_lock(&cgroup_mutex);
-+ offline_css(css);
-+ mutex_unlock(&cgroup_mutex);
-+
-+ css_put(css);
-+}
-+
-+/* css kill confirmation processing requires process context, bounce */
-+static void css_killed_ref_fn(struct percpu_ref *ref)
-+{
-+ struct cgroup_subsys_state *css =
-+ container_of(ref, struct cgroup_subsys_state, refcnt);
-+
-+ INIT_WORK(&css->destroy_work, css_killed_work_fn);
-+ queue_work(cgroup_destroy_wq, &css->destroy_work);
-+}
-+
-+/**
-+ * kill_css - destroy a css
-+ * @css: css to destroy
-+ *
-+ * This function initiates destruction of @css by removing cgroup interface
-+ * files and putting its base reference. ->css_offline() will be invoked
-+ * asynchronously once css_tryget_online() is guaranteed to fail and when
-+ * the reference count reaches zero, @css will be released.
-+ */
-+static void kill_css(struct cgroup_subsys_state *css)
-+{
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ /*
-+ * This must happen before css is disassociated with its cgroup.
-+ * See seq_css() for details.
-+ */
-+ cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
-+
-+ /*
-+ * Killing would put the base ref, but we need to keep it alive
-+ * until after ->css_offline().
-+ */
-+ css_get(css);
-+
-+ /*
-+ * cgroup core guarantees that, by the time ->css_offline() is
-+ * invoked, no new css reference will be given out via
-+ * css_tryget_online(). We can't simply call percpu_ref_kill() and
-+ * proceed to offlining css's because percpu_ref_kill() doesn't
-+ * guarantee that the ref is seen as killed on all CPUs on return.
-+ *
-+ * Use percpu_ref_kill_and_confirm() to get notifications as each
-+ * css is confirmed to be seen as killed on all CPUs.
-+ */
-+ percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
-+}
-+
-+/**
-+ * cgroup_destroy_locked - the first stage of cgroup destruction
-+ * @cgrp: cgroup to be destroyed
-+ *
-+ * css's make use of percpu refcnts whose killing latency shouldn't be
-+ * exposed to userland and are RCU protected. Also, cgroup core needs to
-+ * guarantee that css_tryget_online() won't succeed by the time
-+ * ->css_offline() is invoked. To satisfy all the requirements,
-+ * destruction is implemented in the following two steps.
-+ *
-+ * s1. Verify @cgrp can be destroyed and mark it dying. Remove all
-+ * userland visible parts and start killing the percpu refcnts of
-+ * css's. Set up so that the next stage will be kicked off once all
-+ * the percpu refcnts are confirmed to be killed.
-+ *
-+ * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
-+ * rest of destruction. Once all cgroup references are gone, the
-+ * cgroup is RCU-freed.
-+ *
-+ * This function implements s1. After this step, @cgrp is gone as far as
-+ * the userland is concerned and a new cgroup with the same name may be
-+ * created. As cgroup doesn't care about the names internally, this
-+ * doesn't cause any problem.
-+ */
-+static int cgroup_destroy_locked(struct cgroup *cgrp)
-+ __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
-+{
-+ struct cgroup_subsys_state *css;
-+ bool empty;
-+ int ssid;
-+
-+ lockdep_assert_held(&cgroup_mutex);
-+
-+ /*
-+ * css_set_rwsem synchronizes access to ->cset_links and prevents
-+ * @cgrp from being removed while put_css_set() is in progress.
-+ */
-+ down_read(&css_set_rwsem);
-+ empty = list_empty(&cgrp->cset_links);
-+ up_read(&css_set_rwsem);
-+ if (!empty)
-+ return -EBUSY;
-+
-+ /*
-+ * Make sure there's no live children. We can't test emptiness of
-+ * ->self.children as dead children linger on it while being
-+ * drained; otherwise, "rmdir parent/child parent" may fail.
-+ */
-+ if (css_has_online_children(&cgrp->self))
-+ return -EBUSY;
-+
-+ /*
-+ * Mark @cgrp dead. This prevents further task migration and child
-+ * creation by disabling cgroup_lock_live_group().
-+ */
-+ cgrp->self.flags &= ~CSS_ONLINE;
-+
-+ /* initiate massacre of all css's */
-+ for_each_css(css, ssid, cgrp)
-+ kill_css(css);
-+
-+ /*
-+ * Remove @cgrp directory along with the base files. @cgrp has an
-+ * extra ref on its kn.
-+ */
-+ kernfs_remove(cgrp->kn);
-+
-+ check_for_release(cgroup_parent(cgrp));
-+
-+ /* put the base reference */
-+ percpu_ref_kill(&cgrp->self.refcnt);
-+
-+ return 0;
-+};
-+
-+static int cgroup_rmdir(struct kernfs_node *kn)
-+{
-+ struct cgroup *cgrp;
-+ int ret = 0;
-+
-+ cgrp = cgroup_kn_lock_live(kn);
-+ if (!cgrp)
-+ return 0;
-+
-+ ret = cgroup_destroy_locked(cgrp);
-+
-+ cgroup_kn_unlock(kn);
-+ return ret;
-+}
-+
-+static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
-+ .remount_fs = cgroup_remount,
-+ .show_options = cgroup_show_options,
-+ .mkdir = cgroup_mkdir,
-+ .rmdir = cgroup_rmdir,
-+ .rename = cgroup_rename,
-+};
-+
-+static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
-+{
-+ struct cgroup_subsys_state *css;
-+
-+ printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ idr_init(&ss->css_idr);
-+ INIT_LIST_HEAD(&ss->cfts);
-+
-+ /* Create the root cgroup state for this subsystem */
-+ ss->root = &cgrp_dfl_root;
-+ css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
-+ /* We don't handle early failures gracefully */
-+ BUG_ON(IS_ERR(css));
-+ init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
-+
-+ /*
-+ * Root csses are never destroyed and we can't initialize
-+ * percpu_ref during early init. Disable refcnting.
-+ */
-+ css->flags |= CSS_NO_REF;
-+
-+ if (early) {
-+ /* allocation can't be done safely during early init */
-+ css->id = 1;
-+ } else {
-+ css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
-+ BUG_ON(css->id < 0);
-+ }
-+
-+ /* Update the init_css_set to contain a subsys
-+ * pointer to this state - since the subsystem is
-+ * newly registered, all tasks and hence the
-+ * init_css_set is in the subsystem's root cgroup. */
-+ init_css_set.subsys[ss->id] = css;
-+
-+ need_forkexit_callback |= ss->fork || ss->exit;
-+
-+ /* At system boot, before all subsystems have been
-+ * registered, no tasks have been forked, so we don't
-+ * need to invoke fork callbacks here. */
-+ BUG_ON(!list_empty(&init_task.tasks));
-+
-+ BUG_ON(online_css(css));
-+
-+ mutex_unlock(&cgroup_mutex);
-+}
-+
-+/**
-+ * cgroup_init_early - cgroup initialization at system boot
-+ *
-+ * Initialize cgroups at system boot, and initialize any
-+ * subsystems that request early init.
-+ */
-+int __init cgroup_init_early(void)
-+{
-+ static struct cgroup_sb_opts __initdata opts;
-+ struct cgroup_subsys *ss;
-+ int i;
-+
-+ init_cgroup_root(&cgrp_dfl_root, &opts);
-+ cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF;
-+
-+ RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
-+
-+ for_each_subsys(ss, i) {
-+ WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
-+ "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n",
-+ i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
-+ ss->id, ss->name);
-+ WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
-+ "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);
-+
-+ ss->id = i;
-+ ss->name = cgroup_subsys_name[i];
-+
-+ if (ss->early_init)
-+ cgroup_init_subsys(ss, true);
-+ }
-+ return 0;
-+}
-+
-+/**
-+ * cgroup_init - cgroup initialization
-+ *
-+ * Register cgroup filesystem and /proc file, and initialize
-+ * any subsystems that didn't request early init.
-+ */
-+int __init cgroup_init(void)
-+{
-+ struct cgroup_subsys *ss;
-+ unsigned long key;
-+ int ssid, err;
-+
-+ BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files));
-+ BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files));
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ /* Add init_css_set to the hash table */
-+ key = css_set_hash(init_css_set.subsys);
-+ hash_add(css_set_table, &init_css_set.hlist, key);
-+
-+ BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0));
-+
-+ mutex_unlock(&cgroup_mutex);
-+
-+ for_each_subsys(ss, ssid) {
-+ if (ss->early_init) {
-+ struct cgroup_subsys_state *css =
-+ init_css_set.subsys[ss->id];
-+
-+ css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
-+ GFP_KERNEL);
-+ BUG_ON(css->id < 0);
-+ } else {
-+ cgroup_init_subsys(ss, false);
-+ }
-+
-+ list_add_tail(&init_css_set.e_cset_node[ssid],
-+ &cgrp_dfl_root.cgrp.e_csets[ssid]);
-+
-+ /*
-+ * Setting dfl_root subsys_mask needs to consider the
-+ * disabled flag and cftype registration needs kmalloc,
-+ * both of which aren't available during early_init.
-+ */
-+ if (ss->disabled)
-+ continue;
-+
-+ cgrp_dfl_root.subsys_mask |= 1 << ss->id;
-+
-+ if (cgroup_legacy_files_on_dfl && !ss->dfl_cftypes)
-+ ss->dfl_cftypes = ss->legacy_cftypes;
-+
-+ if (!ss->dfl_cftypes)
-+ cgrp_dfl_root_inhibit_ss_mask |= 1 << ss->id;
-+
-+ if (ss->dfl_cftypes == ss->legacy_cftypes) {
-+ WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));
-+ } else {
-+ WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));
-+ WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));
-+ }
-+
-+ if (ss->bind)
-+ ss->bind(init_css_set.subsys[ssid]);
-+ }
-+
-+ err = sysfs_create_mount_point(fs_kobj, "cgroup");
-+ if (err)
-+ return err;
-+
-+ err = register_filesystem(&cgroup_fs_type);
-+ if (err < 0) {
-+ sysfs_remove_mount_point(fs_kobj, "cgroup");
-+ return err;
-+ }
-+
-+ proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations);
-+ return 0;
-+}
-+
-+static int __init cgroup_wq_init(void)
-+{
-+ /*
-+ * There isn't much point in executing destruction path in
-+ * parallel. Good chunk is serialized with cgroup_mutex anyway.
-+ * Use 1 for @max_active.
-+ *
-+ * We would prefer to do this in cgroup_init() above, but that
-+ * is called before init_workqueues(): so leave this until after.
-+ */
-+ cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
-+ BUG_ON(!cgroup_destroy_wq);
-+
-+ /*
-+ * Used to destroy pidlists and separate to serve as flush domain.
-+ * Cap @max_active to 1 too.
-+ */
-+ cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
-+ 0, 1);
-+ BUG_ON(!cgroup_pidlist_destroy_wq);
-+
-+ return 0;
-+}
-+core_initcall(cgroup_wq_init);
-+
-+/*
-+ * proc_cgroup_show()
-+ * - Print task's cgroup paths into seq_file, one line for each hierarchy
-+ * - Used for /proc/<pid>/cgroup.
-+ */
-+int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
-+ struct pid *pid, struct task_struct *tsk)
-+{
-+ char *buf, *path;
-+ int retval;
-+ struct cgroup_root *root;
-+
-+ retval = -ENOMEM;
-+ buf = kmalloc(PATH_MAX, GFP_KERNEL);
-+ if (!buf)
-+ goto out;
-+
-+ mutex_lock(&cgroup_mutex);
-+ down_read(&css_set_rwsem);
-+
-+ for_each_root(root) {
-+ struct cgroup_subsys *ss;
-+ struct cgroup *cgrp;
-+ int ssid, count = 0;
-+
-+ if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible)
-+ continue;
-+
-+ seq_printf(m, "%d:", root->hierarchy_id);
-+ for_each_subsys(ss, ssid)
-+ if (root->subsys_mask & (1 << ssid))
-+ seq_printf(m, "%s%s", count++ ? "," : "", ss->name);
-+ if (strlen(root->name))
-+ seq_printf(m, "%sname=%s", count ? "," : "",
-+ root->name);
-+ seq_putc(m, ':');
-+ cgrp = task_cgroup_from_root(tsk, root);
-+ path = cgroup_path(cgrp, buf, PATH_MAX);
-+ if (!path) {
-+ retval = -ENAMETOOLONG;
-+ goto out_unlock;
-+ }
-+ seq_puts(m, path);
-+ seq_putc(m, '\n');
-+ }
-+
-+ retval = 0;
-+out_unlock:
-+ up_read(&css_set_rwsem);
-+ mutex_unlock(&cgroup_mutex);
-+ kfree(buf);
-+out:
-+ return retval;
-+}
-+
-+/* Display information about each subsystem and each hierarchy */
-+static int proc_cgroupstats_show(struct seq_file *m, void *v)
-+{
-+ struct cgroup_subsys *ss;
-+ int i;
-+
-+ seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
-+ /*
-+ * ideally we don't want subsystems moving around while we do this.
-+ * cgroup_mutex is also necessary to guarantee an atomic snapshot of
-+ * subsys/hierarchy state.
-+ */
-+ mutex_lock(&cgroup_mutex);
-+
-+ for_each_subsys(ss, i)
-+ seq_printf(m, "%s\t%d\t%d\t%d\n",
-+ ss->name, ss->root->hierarchy_id,
-+ atomic_read(&ss->root->nr_cgrps), !ss->disabled);
-+
-+ mutex_unlock(&cgroup_mutex);
-+ return 0;
-+}
-+
-+static int cgroupstats_open(struct inode *inode, struct file *file)
-+{
-+ return single_open(file, proc_cgroupstats_show, NULL);
-+}
-+
-+static const struct file_operations proc_cgroupstats_operations = {
-+ .open = cgroupstats_open,
-+ .read = seq_read,
-+ .llseek = seq_lseek,
-+ .release = single_release,
-+};
-+
-+/**
-+ * cgroup_fork - initialize cgroup related fields during copy_process()
-+ * @child: pointer to task_struct of forking parent process.
-+ *
-+ * A task is associated with the init_css_set until cgroup_post_fork()
-+ * attaches it to the parent's css_set. Empty cg_list indicates that
-+ * @child isn't holding reference to its css_set.
-+ */
-+void cgroup_fork(struct task_struct *child)
-+{
-+ RCU_INIT_POINTER(child->cgroups, &init_css_set);
-+ INIT_LIST_HEAD(&child->cg_list);
-+}
-+
-+/**
-+ * cgroup_post_fork - called on a new task after adding it to the task list
-+ * @child: the task in question
-+ *
-+ * Adds the task to the list running through its css_set if necessary and
-+ * call the subsystem fork() callbacks. Has to be after the task is
-+ * visible on the task list in case we race with the first call to
-+ * cgroup_task_iter_start() - to guarantee that the new task ends up on its
-+ * list.
-+ */
-+void cgroup_post_fork(struct task_struct *child)
-+{
-+ struct cgroup_subsys *ss;
-+ int i;
-+
-+ /*
-+ * This may race against cgroup_enable_task_cg_lists(). As that
-+ * function sets use_task_css_set_links before grabbing
-+ * tasklist_lock and we just went through tasklist_lock to add
-+ * @child, it's guaranteed that either we see the set
-+ * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
-+ * @child during its iteration.
-+ *
-+ * If we won the race, @child is associated with %current's
-+ * css_set. Grabbing css_set_rwsem guarantees both that the
-+ * association is stable, and, on completion of the parent's
-+ * migration, @child is visible in the source of migration or
-+ * already in the destination cgroup. This guarantee is necessary
-+ * when implementing operations which need to migrate all tasks of
-+ * a cgroup to another.
-+ *
-+ * Note that if we lose to cgroup_enable_task_cg_lists(), @child
-+ * will remain in init_css_set. This is safe because all tasks are
-+ * in the init_css_set before cg_links is enabled and there's no
-+ * operation which transfers all tasks out of init_css_set.
-+ */
-+ if (use_task_css_set_links) {
-+ struct css_set *cset;
-+
-+ down_write(&css_set_rwsem);
-+ cset = task_css_set(current);
-+ if (list_empty(&child->cg_list)) {
-+ rcu_assign_pointer(child->cgroups, cset);
-+ list_add(&child->cg_list, &cset->tasks);
-+ get_css_set(cset);
-+ }
-+ up_write(&css_set_rwsem);
-+ }
-+
-+ /*
-+ * Call ss->fork(). This must happen after @child is linked on
-+ * css_set; otherwise, @child might change state between ->fork()
-+ * and addition to css_set.
-+ */
-+ if (need_forkexit_callback) {
-+ for_each_subsys(ss, i)
-+ if (ss->fork)
-+ ss->fork(child);
-+ }
-+}
-+
-+/**
-+ * cgroup_exit - detach cgroup from exiting task
-+ * @tsk: pointer to task_struct of exiting process
-+ *
-+ * Description: Detach cgroup from @tsk and release it.
-+ *
-+ * Note that cgroups marked notify_on_release force every task in
-+ * them to take the global cgroup_mutex mutex when exiting.
-+ * This could impact scaling on very large systems. Be reluctant to
-+ * use notify_on_release cgroups where very high task exit scaling
-+ * is required on large systems.
-+ *
-+ * We set the exiting tasks cgroup to the root cgroup (top_cgroup). We
-+ * call cgroup_exit() while the task is still competent to handle
-+ * notify_on_release(), then leave the task attached to the root cgroup in
-+ * each hierarchy for the remainder of its exit. No need to bother with
-+ * init_css_set refcnting. init_css_set never goes away and we can't race
-+ * with migration path - PF_EXITING is visible to migration path.
-+ */
-+void cgroup_exit(struct task_struct *tsk)
-+{
-+ struct cgroup_subsys *ss;
-+ struct css_set *cset;
-+ bool put_cset = false;
-+ int i;
-+
-+ /*
-+ * Unlink from @tsk from its css_set. As migration path can't race
-+ * with us, we can check cg_list without grabbing css_set_rwsem.
-+ */
-+ if (!list_empty(&tsk->cg_list)) {
-+ down_write(&css_set_rwsem);
-+ list_del_init(&tsk->cg_list);
-+ up_write(&css_set_rwsem);
-+ put_cset = true;
-+ }
-+
-+ /* Reassign the task to the init_css_set. */
-+ cset = task_css_set(tsk);
-+ RCU_INIT_POINTER(tsk->cgroups, &init_css_set);
-+
-+ if (need_forkexit_callback) {
-+ /* see cgroup_post_fork() for details */
-+ for_each_subsys(ss, i) {
-+ if (ss->exit) {
-+ struct cgroup_subsys_state *old_css = cset->subsys[i];
-+ struct cgroup_subsys_state *css = task_css(tsk, i);
-+
-+ ss->exit(css, old_css, tsk);
-+ }
-+ }
-+ }
-+
-+ if (put_cset)
-+ put_css_set(cset);
-+}
-+
-+static void check_for_release(struct cgroup *cgrp)
-+{
-+ if (notify_on_release(cgrp) && !cgroup_has_tasks(cgrp) &&
-+ !css_has_online_children(&cgrp->self) && !cgroup_is_dead(cgrp))
-+ schedule_work(&cgrp->release_agent_work);
-+}
-+
-+/*
-+ * Notify userspace when a cgroup is released, by running the
-+ * configured release agent with the name of the cgroup (path
-+ * relative to the root of cgroup file system) as the argument.
-+ *
-+ * Most likely, this user command will try to rmdir this cgroup.
-+ *
-+ * This races with the possibility that some other task will be
-+ * attached to this cgroup before it is removed, or that some other
-+ * user task will 'mkdir' a child cgroup of this cgroup. That's ok.
-+ * The presumed 'rmdir' will fail quietly if this cgroup is no longer
-+ * unused, and this cgroup will be reprieved from its death sentence,
-+ * to continue to serve a useful existence. Next time it's released,
-+ * we will get notified again, if it still has 'notify_on_release' set.
-+ *
-+ * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
-+ * means only wait until the task is successfully execve()'d. The
-+ * separate release agent task is forked by call_usermodehelper(),
-+ * then control in this thread returns here, without waiting for the
-+ * release agent task. We don't bother to wait because the caller of
-+ * this routine has no use for the exit status of the release agent
-+ * task, so no sense holding our caller up for that.
-+ */
-+static void cgroup_release_agent(struct work_struct *work)
-+{
-+ struct cgroup *cgrp =
-+ container_of(work, struct cgroup, release_agent_work);
-+ char *pathbuf = NULL, *agentbuf = NULL, *path;
-+ char *argv[3], *envp[3];
-+
-+ mutex_lock(&cgroup_mutex);
-+
-+ pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
-+ agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
-+ if (!pathbuf || !agentbuf)
-+ goto out;
-+
-+ path = cgroup_path(cgrp, pathbuf, PATH_MAX);
-+ if (!path)
-+ goto out;
-+
-+ argv[0] = agentbuf;
-+ argv[1] = path;
-+ argv[2] = NULL;
-+
-+ /* minimal command environment */
-+ envp[0] = "HOME=/";
-+ envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
-+ envp[2] = NULL;
-+
-+ mutex_unlock(&cgroup_mutex);
-+ call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
-+ goto out_free;
-+out:
-+ mutex_unlock(&cgroup_mutex);
-+out_free:
-+ kfree(agentbuf);
-+ kfree(pathbuf);
-+}
-+
-+static int __init cgroup_disable(char *str)
-+{
-+ struct cgroup_subsys *ss;
-+ char *token;
-+ int i;
-+
-+ while ((token = strsep(&str, ",")) != NULL) {
-+ if (!*token)
-+ continue;
-+
-+ for_each_subsys(ss, i) {
-+ if (!strcmp(token, ss->name)) {
-+ ss->disabled = 1;
-+ printk(KERN_INFO "Disabling %s control group"
-+ " subsystem\n", ss->name);
-+ break;
-+ }
-+ }
-+ }
-+ return 1;
-+}
-+__setup("cgroup_disable=", cgroup_disable);
-+
-+static int __init cgroup_set_legacy_files_on_dfl(char *str)
-+{
-+ printk("cgroup: using legacy files on the default hierarchy\n");
-+ cgroup_legacy_files_on_dfl = true;
-+ return 0;
-+}
-+__setup("cgroup__DEVEL__legacy_files_on_dfl", cgroup_set_legacy_files_on_dfl);
-+
-+/**
-+ * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
-+ * @dentry: directory dentry of interest
-+ * @ss: subsystem of interest
-+ *
-+ * If @dentry is a directory for a cgroup which has @ss enabled on it, try
-+ * to get the corresponding css and return it. If such css doesn't exist
-+ * or can't be pinned, an ERR_PTR value is returned.
-+ */
-+struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
-+ struct cgroup_subsys *ss)
-+{
-+ struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
-+ struct cgroup_subsys_state *css = NULL;
-+ struct cgroup *cgrp;
-+
-+ /* is @dentry a cgroup dir? */
-+ if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
-+ kernfs_type(kn) != KERNFS_DIR)
-+ return ERR_PTR(-EBADF);
-+
-+ rcu_read_lock();
-+
-+ /*
-+ * This path doesn't originate from kernfs and @kn could already
-+ * have been or be removed at any point. @kn->priv is RCU
-+ * protected for this access. See css_release_work_fn() for details.
-+ */
-+ cgrp = rcu_dereference(kn->priv);
-+ if (cgrp)
-+ css = cgroup_css(cgrp, ss);
-+
-+ if (!css || !css_tryget_online(css))
-+ css = ERR_PTR(-ENOENT);
-+
-+ rcu_read_unlock();
-+ return css;
-+}
-+
-+/**
-+ * css_from_id - lookup css by id
-+ * @id: the cgroup id
-+ * @ss: cgroup subsys to be looked into
-+ *
-+ * Returns the css if there's valid one with @id, otherwise returns NULL.
-+ * Should be called under rcu_read_lock().
-+ */
-+struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
-+{
-+ WARN_ON_ONCE(!rcu_read_lock_held());
-+ return id > 0 ? idr_find(&ss->css_idr, id) : NULL;
-+}
-+
-+#ifdef CONFIG_CGROUP_DEBUG
-+static struct cgroup_subsys_state *
-+debug_css_alloc(struct cgroup_subsys_state *parent_css)
-+{
-+ struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);
-+
-+ if (!css)
-+ return ERR_PTR(-ENOMEM);
-+
-+ return css;
-+}
-+
-+static void debug_css_free(struct cgroup_subsys_state *css)
-+{
-+ kfree(css);
-+}
-+
-+static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ return cgroup_task_count(css->cgroup);
-+}
-+
-+static u64 current_css_set_read(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ return (u64)(unsigned long)current->cgroups;
-+}
-+
-+static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ u64 count;
-+
-+ rcu_read_lock();
-+ count = atomic_read(&task_css_set(current)->refcount);
-+ rcu_read_unlock();
-+ return count;
-+}
-+
-+static int current_css_set_cg_links_read(struct seq_file *seq, void *v)
-+{
-+ struct cgrp_cset_link *link;
-+ struct css_set *cset;
-+ char *name_buf;
-+
-+ name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL);
-+ if (!name_buf)
-+ return -ENOMEM;
-+
-+ down_read(&css_set_rwsem);
-+ rcu_read_lock();
-+ cset = rcu_dereference(current->cgroups);
-+ list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
-+ struct cgroup *c = link->cgrp;
-+
-+ cgroup_name(c, name_buf, NAME_MAX + 1);
-+ seq_printf(seq, "Root %d group %s\n",
-+ c->root->hierarchy_id, name_buf);
-+ }
-+ rcu_read_unlock();
-+ up_read(&css_set_rwsem);
-+ kfree(name_buf);
-+ return 0;
-+}
-+
-+#define MAX_TASKS_SHOWN_PER_CSS 25
-+static int cgroup_css_links_read(struct seq_file *seq, void *v)
-+{
-+ struct cgroup_subsys_state *css = seq_css(seq);
-+ struct cgrp_cset_link *link;
-+
-+ down_read(&css_set_rwsem);
-+ list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
-+ struct css_set *cset = link->cset;
-+ struct task_struct *task;
-+ int count = 0;
-+
-+ seq_printf(seq, "css_set %p\n", cset);
-+
-+ list_for_each_entry(task, &cset->tasks, cg_list) {
-+ if (count++ > MAX_TASKS_SHOWN_PER_CSS)
-+ goto overflow;
-+ seq_printf(seq, " task %d\n", task_pid_vnr(task));
-+ }
-+
-+ list_for_each_entry(task, &cset->mg_tasks, cg_list) {
-+ if (count++ > MAX_TASKS_SHOWN_PER_CSS)
-+ goto overflow;
-+ seq_printf(seq, " task %d\n", task_pid_vnr(task));
-+ }
-+ continue;
-+ overflow:
-+ seq_puts(seq, " ...\n");
-+ }
-+ up_read(&css_set_rwsem);
-+ return 0;
-+}
-+
-+static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
-+{
-+ return (!cgroup_has_tasks(css->cgroup) &&
-+ !css_has_online_children(&css->cgroup->self));
-+}
-+
-+static struct cftype debug_files[] = {
-+ {
-+ .name = "taskcount",
-+ .read_u64 = debug_taskcount_read,
-+ },
-+
-+ {
-+ .name = "current_css_set",
-+ .read_u64 = current_css_set_read,
-+ },
-+
-+ {
-+ .name = "current_css_set_refcount",
-+ .read_u64 = current_css_set_refcount_read,
-+ },
-+
-+ {
-+ .name = "current_css_set_cg_links",
-+ .seq_show = current_css_set_cg_links_read,
-+ },
-+
-+ {
-+ .name = "cgroup_css_links",
-+ .seq_show = cgroup_css_links_read,
-+ },
-+
-+ {
-+ .name = "releasable",
-+ .read_u64 = releasable_read,
-+ },
-+
-+ { } /* terminate */
-+};
-+
-+struct cgroup_subsys debug_cgrp_subsys = {
-+ .css_alloc = debug_css_alloc,
-+ .css_free = debug_css_free,
-+ .legacy_cftypes = debug_files,
-+};
-+#endif /* CONFIG_CGROUP_DEBUG */
diff -Nur linux-4.1.10.orig/kernel/cpu.c linux-4.1.10/kernel/cpu.c
--- linux-4.1.10.orig/kernel/cpu.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/cpu.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/cpu.c 2015-10-12 22:33:32.292675697 +0200
@@ -74,8 +74,8 @@
#endif
} cpu_hotplug = {
@@ -29216,7 +13810,7 @@ diff -Nur linux-4.1.10.orig/kernel/cpu.c linux-4.1.10/kernel/cpu.c
diff -Nur linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c linux-4.1.10/kernel/debug/kdb/kdb_io.c
--- linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/debug/kdb/kdb_io.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/debug/kdb/kdb_io.c 2015-10-12 22:33:32.292675697 +0200
@@ -554,7 +554,6 @@
int linecount;
int colcount;
@@ -29256,7 +13850,7 @@ diff -Nur linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c linux-4.1.10/kernel/debug/
}
diff -Nur linux-4.1.10.orig/kernel/events/core.c linux-4.1.10/kernel/events/core.c
--- linux-4.1.10.orig/kernel/events/core.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/events/core.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/events/core.c 2015-10-12 22:33:32.292675697 +0200
@@ -6933,6 +6933,7 @@
hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
@@ -29267,7 +13861,7 @@ diff -Nur linux-4.1.10.orig/kernel/events/core.c linux-4.1.10/kernel/events/core
* Since hrtimers have a fixed rate, we can do a static freq->period
diff -Nur linux-4.1.10.orig/kernel/exit.c linux-4.1.10/kernel/exit.c
--- linux-4.1.10.orig/kernel/exit.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/exit.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/exit.c 2015-10-12 22:33:32.292675697 +0200
@@ -144,7 +144,7 @@
* Do this under ->siglock, we can race with another thread
* doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals.
@@ -29279,7 +13873,7 @@ diff -Nur linux-4.1.10.orig/kernel/exit.c linux-4.1.10/kernel/exit.c
diff -Nur linux-4.1.10.orig/kernel/fork.c linux-4.1.10/kernel/fork.c
--- linux-4.1.10.orig/kernel/fork.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/fork.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/fork.c 2015-10-12 22:33:32.292675697 +0200
@@ -108,7 +108,7 @@
DEFINE_PER_CPU(unsigned long, process_counts) = 0;
@@ -29379,7 +13973,7 @@ diff -Nur linux-4.1.10.orig/kernel/fork.c linux-4.1.10/kernel/fork.c
p->curr_chain_key = 0;
diff -Nur linux-4.1.10.orig/kernel/futex.c linux-4.1.10/kernel/futex.c
--- linux-4.1.10.orig/kernel/futex.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/futex.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/futex.c 2015-10-12 22:33:32.292675697 +0200
@@ -738,7 +738,9 @@
* task still owns the PI-state:
*/
@@ -29710,7 +14304,7 @@ diff -Nur linux-4.1.10.orig/kernel/futex.c linux-4.1.10/kernel/futex.c
* haven't already.
diff -Nur linux-4.1.10.orig/kernel/irq/handle.c linux-4.1.10/kernel/irq/handle.c
--- linux-4.1.10.orig/kernel/irq/handle.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/irq/handle.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/irq/handle.c 2015-10-12 22:33:32.296675432 +0200
@@ -133,6 +133,8 @@
irqreturn_t
handle_irq_event_percpu(struct irq_desc *desc, struct irqaction *action)
@@ -29735,7 +14329,7 @@ diff -Nur linux-4.1.10.orig/kernel/irq/handle.c linux-4.1.10/kernel/irq/handle.c
note_interrupt(irq, desc, retval);
diff -Nur linux-4.1.10.orig/kernel/irq/manage.c linux-4.1.10/kernel/irq/manage.c
--- linux-4.1.10.orig/kernel/irq/manage.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/irq/manage.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/irq/manage.c 2015-10-12 22:33:32.296675432 +0200
@@ -22,6 +22,7 @@
#include "internals.h"
@@ -30174,7 +14768,7 @@ diff -Nur linux-4.1.10.orig/kernel/irq/manage.c linux-4.1.10/kernel/irq/manage.c
if (!retval && (irqflags & IRQF_SHARED)) {
diff -Nur linux-4.1.10.orig/kernel/irq/settings.h linux-4.1.10/kernel/irq/settings.h
--- linux-4.1.10.orig/kernel/irq/settings.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/irq/settings.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/irq/settings.h 2015-10-12 22:33:32.296675432 +0200
@@ -15,6 +15,7 @@
_IRQ_NESTED_THREAD = IRQ_NESTED_THREAD,
_IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID,
@@ -30210,7 +14804,7 @@ diff -Nur linux-4.1.10.orig/kernel/irq/settings.h linux-4.1.10/kernel/irq/settin
return desc->status_use_accessors & _IRQ_PER_CPU;
diff -Nur linux-4.1.10.orig/kernel/irq/spurious.c linux-4.1.10/kernel/irq/spurious.c
--- linux-4.1.10.orig/kernel/irq/spurious.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/irq/spurious.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/irq/spurious.c 2015-10-12 22:33:32.296675432 +0200
@@ -444,6 +444,10 @@
static int __init irqfixup_setup(char *str)
@@ -30235,7 +14829,7 @@ diff -Nur linux-4.1.10.orig/kernel/irq/spurious.c linux-4.1.10/kernel/irq/spurio
"enabled\n");
diff -Nur linux-4.1.10.orig/kernel/irq_work.c linux-4.1.10/kernel/irq_work.c
--- linux-4.1.10.orig/kernel/irq_work.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/irq_work.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/irq_work.c 2015-10-12 22:33:32.296675432 +0200
@@ -17,6 +17,7 @@
#include <linux/cpu.h>
#include <linux/notifier.h>
@@ -30341,7 +14935,7 @@ diff -Nur linux-4.1.10.orig/kernel/irq_work.c linux-4.1.10/kernel/irq_work.c
diff -Nur linux-4.1.10.orig/kernel/Kconfig.locks linux-4.1.10/kernel/Kconfig.locks
--- linux-4.1.10.orig/kernel/Kconfig.locks 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/Kconfig.locks 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/Kconfig.locks 2015-10-12 22:33:32.296675432 +0200
@@ -225,11 +225,11 @@
config MUTEX_SPIN_ON_OWNER
@@ -30358,7 +14952,7 @@ diff -Nur linux-4.1.10.orig/kernel/Kconfig.locks linux-4.1.10/kernel/Kconfig.loc
def_bool y
diff -Nur linux-4.1.10.orig/kernel/Kconfig.preempt linux-4.1.10/kernel/Kconfig.preempt
--- linux-4.1.10.orig/kernel/Kconfig.preempt 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/Kconfig.preempt 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/Kconfig.preempt 2015-10-12 22:33:32.296675432 +0200
@@ -1,3 +1,16 @@
+config PREEMPT
+ bool
@@ -30413,7 +15007,7 @@ diff -Nur linux-4.1.10.orig/kernel/Kconfig.preempt linux-4.1.10/kernel/Kconfig.p
config PREEMPT_COUNT
diff -Nur linux-4.1.10.orig/kernel/ksysfs.c linux-4.1.10/kernel/ksysfs.c
--- linux-4.1.10.orig/kernel/ksysfs.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/ksysfs.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/ksysfs.c 2015-10-12 22:33:32.296675432 +0200
@@ -136,6 +136,15 @@
#endif /* CONFIG_KEXEC */
@@ -30442,7 +15036,7 @@ diff -Nur linux-4.1.10.orig/kernel/ksysfs.c linux-4.1.10/kernel/ksysfs.c
diff -Nur linux-4.1.10.orig/kernel/locking/lglock.c linux-4.1.10/kernel/locking/lglock.c
--- linux-4.1.10.orig/kernel/locking/lglock.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/locking/lglock.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/locking/lglock.c 2015-10-12 22:33:32.296675432 +0200
@@ -4,6 +4,15 @@
#include <linux/cpu.h>
#include <linux/string.h>
@@ -30589,7 +15183,7 @@ diff -Nur linux-4.1.10.orig/kernel/locking/lglock.c linux-4.1.10/kernel/locking/
+#endif
diff -Nur linux-4.1.10.orig/kernel/locking/lockdep.c linux-4.1.10/kernel/locking/lockdep.c
--- linux-4.1.10.orig/kernel/locking/lockdep.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/locking/lockdep.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/locking/lockdep.c 2015-10-12 22:33:32.296675432 +0200
@@ -3563,6 +3563,7 @@
}
}
@@ -30608,7 +15202,7 @@ diff -Nur linux-4.1.10.orig/kernel/locking/lockdep.c linux-4.1.10/kernel/locking
print_irqtrace_events(current);
diff -Nur linux-4.1.10.orig/kernel/locking/locktorture.c linux-4.1.10/kernel/locking/locktorture.c
--- linux-4.1.10.orig/kernel/locking/locktorture.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/locking/locktorture.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/locking/locktorture.c 2015-10-12 22:33:32.296675432 +0200
@@ -24,7 +24,6 @@
#include <linux/module.h>
#include <linux/kthread.h>
@@ -30619,7 +15213,7 @@ diff -Nur linux-4.1.10.orig/kernel/locking/locktorture.c linux-4.1.10/kernel/loc
#include <linux/smp.h>
diff -Nur linux-4.1.10.orig/kernel/locking/Makefile linux-4.1.10/kernel/locking/Makefile
--- linux-4.1.10.orig/kernel/locking/Makefile 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/locking/Makefile 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/locking/Makefile 2015-10-12 22:33:32.296675432 +0200
@@ -1,5 +1,5 @@
-obj-y += mutex.o semaphore.o rwsem.o
@@ -30653,7 +15247,7 @@ diff -Nur linux-4.1.10.orig/kernel/locking/Makefile linux-4.1.10/kernel/locking/
obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o
diff -Nur linux-4.1.10.orig/kernel/locking/rt.c linux-4.1.10/kernel/locking/rt.c
--- linux-4.1.10.orig/kernel/locking/rt.c 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/kernel/locking/rt.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/locking/rt.c 2015-10-12 22:33:32.296675432 +0200
@@ -0,0 +1,461 @@
+/*
+ * kernel/rt.c
@@ -31118,7 +15712,7 @@ diff -Nur linux-4.1.10.orig/kernel/locking/rt.c linux-4.1.10/kernel/locking/rt.c
+EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex.c linux-4.1.10/kernel/locking/rtmutex.c
--- linux-4.1.10.orig/kernel/locking/rtmutex.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/locking/rtmutex.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/locking/rtmutex.c 2015-10-12 22:33:32.296675432 +0200
@@ -7,6 +7,11 @@
* Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
* Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
@@ -32209,7 +16803,7 @@ diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex.c linux-4.1.10/kernel/locking
+#endif
diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex_common.h linux-4.1.10/kernel/locking/rtmutex_common.h
--- linux-4.1.10.orig/kernel/locking/rtmutex_common.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/locking/rtmutex_common.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/locking/rtmutex_common.h 2015-10-12 22:33:32.296675432 +0200
@@ -49,6 +49,7 @@
struct rb_node pi_tree_entry;
struct task_struct *task;
@@ -32255,7 +16849,7 @@ diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex_common.h linux-4.1.10/kernel/
#endif
diff -Nur linux-4.1.10.orig/kernel/locking/spinlock.c linux-4.1.10/kernel/locking/spinlock.c
--- linux-4.1.10.orig/kernel/locking/spinlock.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/locking/spinlock.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/locking/spinlock.c 2015-10-12 22:33:32.300675168 +0200
@@ -124,8 +124,11 @@
* __[spin|read|write]_lock_bh()
*/
@@ -32288,7 +16882,7 @@ diff -Nur linux-4.1.10.orig/kernel/locking/spinlock.c linux-4.1.10/kernel/lockin
void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)
diff -Nur linux-4.1.10.orig/kernel/locking/spinlock_debug.c linux-4.1.10/kernel/locking/spinlock_debug.c
--- linux-4.1.10.orig/kernel/locking/spinlock_debug.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/locking/spinlock_debug.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/locking/spinlock_debug.c 2015-10-12 22:33:32.300675168 +0200
@@ -31,6 +31,7 @@
EXPORT_SYMBOL(__raw_spin_lock_init);
@@ -32321,7 +16915,7 @@ diff -Nur linux-4.1.10.orig/kernel/locking/spinlock_debug.c linux-4.1.10/kernel/
+#endif
diff -Nur linux-4.1.10.orig/kernel/panic.c linux-4.1.10/kernel/panic.c
--- linux-4.1.10.orig/kernel/panic.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/panic.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/panic.c 2015-10-12 22:33:32.300675168 +0200
@@ -387,9 +387,11 @@
static int init_oops_id(void)
@@ -32336,7 +16930,7 @@ diff -Nur linux-4.1.10.orig/kernel/panic.c linux-4.1.10/kernel/panic.c
return 0;
diff -Nur linux-4.1.10.orig/kernel/power/hibernate.c linux-4.1.10/kernel/power/hibernate.c
--- linux-4.1.10.orig/kernel/power/hibernate.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/power/hibernate.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/power/hibernate.c 2015-10-12 22:33:32.300675168 +0200
@@ -285,6 +285,8 @@
local_irq_disable();
@@ -32388,7 +16982,7 @@ diff -Nur linux-4.1.10.orig/kernel/power/hibernate.c linux-4.1.10/kernel/power/h
diff -Nur linux-4.1.10.orig/kernel/power/suspend.c linux-4.1.10/kernel/power/suspend.c
--- linux-4.1.10.orig/kernel/power/suspend.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/power/suspend.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/power/suspend.c 2015-10-12 22:33:32.300675168 +0200
@@ -356,6 +356,8 @@
arch_suspend_disable_irqs();
BUG_ON(!irqs_disabled());
@@ -32409,7 +17003,7 @@ diff -Nur linux-4.1.10.orig/kernel/power/suspend.c linux-4.1.10/kernel/power/sus
diff -Nur linux-4.1.10.orig/kernel/printk/printk.c linux-4.1.10/kernel/printk/printk.c
--- linux-4.1.10.orig/kernel/printk/printk.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/printk/printk.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/printk/printk.c 2015-10-12 22:33:32.300675168 +0200
@@ -1163,6 +1163,7 @@
{
char *text;
@@ -32664,7 +17258,7 @@ diff -Nur linux-4.1.10.orig/kernel/printk/printk.c linux-4.1.10/kernel/printk/pr
diff -Nur linux-4.1.10.orig/kernel/ptrace.c linux-4.1.10/kernel/ptrace.c
--- linux-4.1.10.orig/kernel/ptrace.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/ptrace.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/ptrace.c 2015-10-12 22:33:32.300675168 +0200
@@ -129,7 +129,12 @@
spin_lock_irq(&task->sighand->siglock);
@@ -32681,7 +17275,7 @@ diff -Nur linux-4.1.10.orig/kernel/ptrace.c linux-4.1.10/kernel/ptrace.c
spin_unlock_irq(&task->sighand->siglock);
diff -Nur linux-4.1.10.orig/kernel/rcu/tree.c linux-4.1.10/kernel/rcu/tree.c
--- linux-4.1.10.orig/kernel/rcu/tree.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/rcu/tree.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/rcu/tree.c 2015-10-12 22:33:32.300675168 +0200
@@ -56,6 +56,11 @@
#include <linux/random.h>
#include <linux/ftrace_event.h>
@@ -32971,7 +17565,7 @@ diff -Nur linux-4.1.10.orig/kernel/rcu/tree.c linux-4.1.10/kernel/rcu/tree.c
* We don't need protection against CPU-hotplug here because
diff -Nur linux-4.1.10.orig/kernel/rcu/tree.h linux-4.1.10/kernel/rcu/tree.h
--- linux-4.1.10.orig/kernel/rcu/tree.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/rcu/tree.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/rcu/tree.h 2015-10-12 22:33:32.300675168 +0200
@@ -27,6 +27,7 @@
#include <linux/threads.h>
#include <linux/cpumask.h>
@@ -33034,7 +17628,7 @@ diff -Nur linux-4.1.10.orig/kernel/rcu/tree.h linux-4.1.10/kernel/rcu/tree.h
#endif /* #ifdef CONFIG_RCU_BOOST */
diff -Nur linux-4.1.10.orig/kernel/rcu/tree_plugin.h linux-4.1.10/kernel/rcu/tree_plugin.h
--- linux-4.1.10.orig/kernel/rcu/tree_plugin.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/rcu/tree_plugin.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/rcu/tree_plugin.h 2015-10-12 22:33:32.300675168 +0200
@@ -24,27 +24,20 @@
* Paul E. McKenney <paulmck@linux.vnet.ibm.com>
*/
@@ -33371,7 +17965,7 @@ diff -Nur linux-4.1.10.orig/kernel/rcu/tree_plugin.h linux-4.1.10/kernel/rcu/tre
diff -Nur linux-4.1.10.orig/kernel/rcu/update.c linux-4.1.10/kernel/rcu/update.c
--- linux-4.1.10.orig/kernel/rcu/update.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/rcu/update.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/rcu/update.c 2015-10-12 22:33:32.300675168 +0200
@@ -227,6 +227,7 @@
}
EXPORT_SYMBOL_GPL(rcu_read_lock_held);
@@ -33390,7 +17984,7 @@ diff -Nur linux-4.1.10.orig/kernel/rcu/update.c linux-4.1.10/kernel/rcu/update.c
diff -Nur linux-4.1.10.orig/kernel/relay.c linux-4.1.10/kernel/relay.c
--- linux-4.1.10.orig/kernel/relay.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/relay.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/relay.c 2015-10-12 22:33:32.304674904 +0200
@@ -339,6 +339,10 @@
{
struct rchan_buf *buf = (struct rchan_buf *)data;
@@ -33428,7 +18022,7 @@ diff -Nur linux-4.1.10.orig/kernel/relay.c linux-4.1.10/kernel/relay.c
old = buf->data;
diff -Nur linux-4.1.10.orig/kernel/sched/completion.c linux-4.1.10/kernel/sched/completion.c
--- linux-4.1.10.orig/kernel/sched/completion.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/completion.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/completion.c 2015-10-12 22:33:32.304674904 +0200
@@ -30,10 +30,10 @@
{
unsigned long flags;
@@ -33521,7 +18115,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/completion.c linux-4.1.10/kernel/sched/
EXPORT_SYMBOL(completion_done);
diff -Nur linux-4.1.10.orig/kernel/sched/core.c linux-4.1.10/kernel/sched/core.c
--- linux-4.1.10.orig/kernel/sched/core.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/core.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/core.c 2015-10-12 22:33:32.304674904 +0200
@@ -282,7 +282,11 @@
* Number of tasks to iterate in a single balance run.
* Limited because this is done with IRQs disabled.
@@ -34249,8402 +18843,9 @@ diff -Nur linux-4.1.10.orig/kernel/sched/core.c linux-4.1.10/kernel/sched/core.c
return (nested == preempt_offset);
}
-diff -Nur linux-4.1.10.orig/kernel/sched/core.c.orig linux-4.1.10/kernel/sched/core.c.orig
---- linux-4.1.10.orig/kernel/sched/core.c.orig 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/kernel/sched/core.c.orig 2015-10-03 13:49:38.000000000 +0200
-@@ -0,0 +1,8389 @@
-+/*
-+ * kernel/sched/core.c
-+ *
-+ * Kernel scheduler and related syscalls
-+ *
-+ * Copyright (C) 1991-2002 Linus Torvalds
-+ *
-+ * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
-+ * make semaphores SMP safe
-+ * 1998-11-19 Implemented schedule_timeout() and related stuff
-+ * by Andrea Arcangeli
-+ * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
-+ * hybrid priority-list and round-robin design with
-+ * an array-switch method of distributing timeslices
-+ * and per-CPU runqueues. Cleanups and useful suggestions
-+ * by Davide Libenzi, preemptible kernel bits by Robert Love.
-+ * 2003-09-03 Interactivity tuning by Con Kolivas.
-+ * 2004-04-02 Scheduler domains code by Nick Piggin
-+ * 2007-04-15 Work begun on replacing all interactivity tuning with a
-+ * fair scheduling design by Con Kolivas.
-+ * 2007-05-05 Load balancing (smp-nice) and other improvements
-+ * by Peter Williams
-+ * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith
-+ * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri
-+ * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,
-+ * Thomas Gleixner, Mike Kravetz
-+ */
-+
-+#include <linux/mm.h>
-+#include <linux/module.h>
-+#include <linux/nmi.h>
-+#include <linux/init.h>
-+#include <linux/uaccess.h>
-+#include <linux/highmem.h>
-+#include <asm/mmu_context.h>
-+#include <linux/interrupt.h>
-+#include <linux/capability.h>
-+#include <linux/completion.h>
-+#include <linux/kernel_stat.h>
-+#include <linux/debug_locks.h>
-+#include <linux/perf_event.h>
-+#include <linux/security.h>
-+#include <linux/notifier.h>
-+#include <linux/profile.h>
-+#include <linux/freezer.h>
-+#include <linux/vmalloc.h>
-+#include <linux/blkdev.h>
-+#include <linux/delay.h>
-+#include <linux/pid_namespace.h>
-+#include <linux/smp.h>
-+#include <linux/threads.h>
-+#include <linux/timer.h>
-+#include <linux/rcupdate.h>
-+#include <linux/cpu.h>
-+#include <linux/cpuset.h>
-+#include <linux/percpu.h>
-+#include <linux/proc_fs.h>
-+#include <linux/seq_file.h>
-+#include <linux/sysctl.h>
-+#include <linux/syscalls.h>
-+#include <linux/times.h>
-+#include <linux/tsacct_kern.h>
-+#include <linux/kprobes.h>
-+#include <linux/delayacct.h>
-+#include <linux/unistd.h>
-+#include <linux/pagemap.h>
-+#include <linux/hrtimer.h>
-+#include <linux/tick.h>
-+#include <linux/debugfs.h>
-+#include <linux/ctype.h>
-+#include <linux/ftrace.h>
-+#include <linux/slab.h>
-+#include <linux/init_task.h>
-+#include <linux/binfmts.h>
-+#include <linux/context_tracking.h>
-+#include <linux/compiler.h>
-+
-+#include <asm/switch_to.h>
-+#include <asm/tlb.h>
-+#include <asm/irq_regs.h>
-+#include <asm/mutex.h>
-+#ifdef CONFIG_PARAVIRT
-+#include <asm/paravirt.h>
-+#endif
-+
-+#include "sched.h"
-+#include "../workqueue_internal.h"
-+#include "../smpboot.h"
-+
-+#define CREATE_TRACE_POINTS
-+#include <trace/events/sched.h>
-+
-+void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
-+{
-+ unsigned long delta;
-+ ktime_t soft, hard, now;
-+
-+ for (;;) {
-+ if (hrtimer_active(period_timer))
-+ break;
-+
-+ now = hrtimer_cb_get_time(period_timer);
-+ hrtimer_forward(period_timer, now, period);
-+
-+ soft = hrtimer_get_softexpires(period_timer);
-+ hard = hrtimer_get_expires(period_timer);
-+ delta = ktime_to_ns(ktime_sub(hard, soft));
-+ __hrtimer_start_range_ns(period_timer, soft, delta,
-+ HRTIMER_MODE_ABS_PINNED, 0);
-+ }
-+}
-+
-+DEFINE_MUTEX(sched_domains_mutex);
-+DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
-+
-+static void update_rq_clock_task(struct rq *rq, s64 delta);
-+
-+void update_rq_clock(struct rq *rq)
-+{
-+ s64 delta;
-+
-+ lockdep_assert_held(&rq->lock);
-+
-+ if (rq->clock_skip_update & RQCF_ACT_SKIP)
-+ return;
-+
-+ delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
-+ if (delta < 0)
-+ return;
-+ rq->clock += delta;
-+ update_rq_clock_task(rq, delta);
-+}
-+
-+/*
-+ * Debugging: various feature bits
-+ */
-+
-+#define SCHED_FEAT(name, enabled) \
-+ (1UL << __SCHED_FEAT_##name) * enabled |
-+
-+const_debug unsigned int sysctl_sched_features =
-+#include "features.h"
-+ 0;
-+
-+#undef SCHED_FEAT
-+
-+#ifdef CONFIG_SCHED_DEBUG
-+#define SCHED_FEAT(name, enabled) \
-+ #name ,
-+
-+static const char * const sched_feat_names[] = {
-+#include "features.h"
-+};
-+
-+#undef SCHED_FEAT
-+
-+static int sched_feat_show(struct seq_file *m, void *v)
-+{
-+ int i;
-+
-+ for (i = 0; i < __SCHED_FEAT_NR; i++) {
-+ if (!(sysctl_sched_features & (1UL << i)))
-+ seq_puts(m, "NO_");
-+ seq_printf(m, "%s ", sched_feat_names[i]);
-+ }
-+ seq_puts(m, "\n");
-+
-+ return 0;
-+}
-+
-+#ifdef HAVE_JUMP_LABEL
-+
-+#define jump_label_key__true STATIC_KEY_INIT_TRUE
-+#define jump_label_key__false STATIC_KEY_INIT_FALSE
-+
-+#define SCHED_FEAT(name, enabled) \
-+ jump_label_key__##enabled ,
-+
-+struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
-+#include "features.h"
-+};
-+
-+#undef SCHED_FEAT
-+
-+static void sched_feat_disable(int i)
-+{
-+ if (static_key_enabled(&sched_feat_keys[i]))
-+ static_key_slow_dec(&sched_feat_keys[i]);
-+}
-+
-+static void sched_feat_enable(int i)
-+{
-+ if (!static_key_enabled(&sched_feat_keys[i]))
-+ static_key_slow_inc(&sched_feat_keys[i]);
-+}
-+#else
-+static void sched_feat_disable(int i) { };
-+static void sched_feat_enable(int i) { };
-+#endif /* HAVE_JUMP_LABEL */
-+
-+static int sched_feat_set(char *cmp)
-+{
-+ int i;
-+ int neg = 0;
-+
-+ if (strncmp(cmp, "NO_", 3) == 0) {
-+ neg = 1;
-+ cmp += 3;
-+ }
-+
-+ for (i = 0; i < __SCHED_FEAT_NR; i++) {
-+ if (strcmp(cmp, sched_feat_names[i]) == 0) {
-+ if (neg) {
-+ sysctl_sched_features &= ~(1UL << i);
-+ sched_feat_disable(i);
-+ } else {
-+ sysctl_sched_features |= (1UL << i);
-+ sched_feat_enable(i);
-+ }
-+ break;
-+ }
-+ }
-+
-+ return i;
-+}
-+
-+static ssize_t
-+sched_feat_write(struct file *filp, const char __user *ubuf,
-+ size_t cnt, loff_t *ppos)
-+{
-+ char buf[64];
-+ char *cmp;
-+ int i;
-+ struct inode *inode;
-+
-+ if (cnt > 63)
-+ cnt = 63;
-+
-+ if (copy_from_user(&buf, ubuf, cnt))
-+ return -EFAULT;
-+
-+ buf[cnt] = 0;
-+ cmp = strstrip(buf);
-+
-+ /* Ensure the static_key remains in a consistent state */
-+ inode = file_inode(filp);
-+ mutex_lock(&inode->i_mutex);
-+ i = sched_feat_set(cmp);
-+ mutex_unlock(&inode->i_mutex);
-+ if (i == __SCHED_FEAT_NR)
-+ return -EINVAL;
-+
-+ *ppos += cnt;
-+
-+ return cnt;
-+}
-+
-+static int sched_feat_open(struct inode *inode, struct file *filp)
-+{
-+ return single_open(filp, sched_feat_show, NULL);
-+}
-+
-+static const struct file_operations sched_feat_fops = {
-+ .open = sched_feat_open,
-+ .write = sched_feat_write,
-+ .read = seq_read,
-+ .llseek = seq_lseek,
-+ .release = single_release,
-+};
-+
-+static __init int sched_init_debug(void)
-+{
-+ debugfs_create_file("sched_features", 0644, NULL, NULL,
-+ &sched_feat_fops);
-+
-+ return 0;
-+}
-+late_initcall(sched_init_debug);
-+#endif /* CONFIG_SCHED_DEBUG */
-+
-+/*
-+ * Number of tasks to iterate in a single balance run.
-+ * Limited because this is done with IRQs disabled.
-+ */
-+const_debug unsigned int sysctl_sched_nr_migrate = 32;
-+
-+/*
-+ * period over which we average the RT time consumption, measured
-+ * in ms.
-+ *
-+ * default: 1s
-+ */
-+const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
-+
-+/*
-+ * period over which we measure -rt task cpu usage in us.
-+ * default: 1s
-+ */
-+unsigned int sysctl_sched_rt_period = 1000000;
-+
-+__read_mostly int scheduler_running;
-+
-+/*
-+ * part of the period that we allow rt tasks to run in us.
-+ * default: 0.95s
-+ */
-+int sysctl_sched_rt_runtime = 950000;
-+
-+/* cpus with isolated domains */
-+cpumask_var_t cpu_isolated_map;
-+
-+/*
-+ * this_rq_lock - lock this runqueue and disable interrupts.
-+ */
-+static struct rq *this_rq_lock(void)
-+ __acquires(rq->lock)
-+{
-+ struct rq *rq;
-+
-+ local_irq_disable();
-+ rq = this_rq();
-+ raw_spin_lock(&rq->lock);
-+
-+ return rq;
-+}
-+
-+#ifdef CONFIG_SCHED_HRTICK
-+/*
-+ * Use HR-timers to deliver accurate preemption points.
-+ */
-+
-+static void hrtick_clear(struct rq *rq)
-+{
-+ if (hrtimer_active(&rq->hrtick_timer))
-+ hrtimer_cancel(&rq->hrtick_timer);
-+}
-+
-+/*
-+ * High-resolution timer tick.
-+ * Runs from hardirq context with interrupts disabled.
-+ */
-+static enum hrtimer_restart hrtick(struct hrtimer *timer)
-+{
-+ struct rq *rq = container_of(timer, struct rq, hrtick_timer);
-+
-+ WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
-+
-+ raw_spin_lock(&rq->lock);
-+ update_rq_clock(rq);
-+ rq->curr->sched_class->task_tick(rq, rq->curr, 1);
-+ raw_spin_unlock(&rq->lock);
-+
-+ return HRTIMER_NORESTART;
-+}
-+
-+#ifdef CONFIG_SMP
-+
-+static int __hrtick_restart(struct rq *rq)
-+{
-+ struct hrtimer *timer = &rq->hrtick_timer;
-+ ktime_t time = hrtimer_get_softexpires(timer);
-+
-+ return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0);
-+}
-+
-+/*
-+ * called from hardirq (IPI) context
-+ */
-+static void __hrtick_start(void *arg)
-+{
-+ struct rq *rq = arg;
-+
-+ raw_spin_lock(&rq->lock);
-+ __hrtick_restart(rq);
-+ rq->hrtick_csd_pending = 0;
-+ raw_spin_unlock(&rq->lock);
-+}
-+
-+/*
-+ * Called to set the hrtick timer state.
-+ *
-+ * called with rq->lock held and irqs disabled
-+ */
-+void hrtick_start(struct rq *rq, u64 delay)
-+{
-+ struct hrtimer *timer = &rq->hrtick_timer;
-+ ktime_t time;
-+ s64 delta;
-+
-+ /*
-+ * Don't schedule slices shorter than 10000ns, that just
-+ * doesn't make sense and can cause timer DoS.
-+ */
-+ delta = max_t(s64, delay, 10000LL);
-+ time = ktime_add_ns(timer->base->get_time(), delta);
-+
-+ hrtimer_set_expires(timer, time);
-+
-+ if (rq == this_rq()) {
-+ __hrtick_restart(rq);
-+ } else if (!rq->hrtick_csd_pending) {
-+ smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
-+ rq->hrtick_csd_pending = 1;
-+ }
-+}
-+
-+static int
-+hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
-+{
-+ int cpu = (int)(long)hcpu;
-+
-+ switch (action) {
-+ case CPU_UP_CANCELED:
-+ case CPU_UP_CANCELED_FROZEN:
-+ case CPU_DOWN_PREPARE:
-+ case CPU_DOWN_PREPARE_FROZEN:
-+ case CPU_DEAD:
-+ case CPU_DEAD_FROZEN:
-+ hrtick_clear(cpu_rq(cpu));
-+ return NOTIFY_OK;
-+ }
-+
-+ return NOTIFY_DONE;
-+}
-+
-+static __init void init_hrtick(void)
-+{
-+ hotcpu_notifier(hotplug_hrtick, 0);
-+}
-+#else
-+/*
-+ * Called to set the hrtick timer state.
-+ *
-+ * called with rq->lock held and irqs disabled
-+ */
-+void hrtick_start(struct rq *rq, u64 delay)
-+{
-+ /*
-+ * Don't schedule slices shorter than 10000ns, that just
-+ * doesn't make sense. Rely on vruntime for fairness.
-+ */
-+ delay = max_t(u64, delay, 10000LL);
-+ __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
-+ HRTIMER_MODE_REL_PINNED, 0);
-+}
-+
-+static inline void init_hrtick(void)
-+{
-+}
-+#endif /* CONFIG_SMP */
-+
-+static void init_rq_hrtick(struct rq *rq)
-+{
-+#ifdef CONFIG_SMP
-+ rq->hrtick_csd_pending = 0;
-+
-+ rq->hrtick_csd.flags = 0;
-+ rq->hrtick_csd.func = __hrtick_start;
-+ rq->hrtick_csd.info = rq;
-+#endif
-+
-+ hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
-+ rq->hrtick_timer.function = hrtick;
-+}
-+#else /* CONFIG_SCHED_HRTICK */
-+static inline void hrtick_clear(struct rq *rq)
-+{
-+}
-+
-+static inline void init_rq_hrtick(struct rq *rq)
-+{
-+}
-+
-+static inline void init_hrtick(void)
-+{
-+}
-+#endif /* CONFIG_SCHED_HRTICK */
-+
-+/*
-+ * cmpxchg based fetch_or, macro so it works for different integer types
-+ */
-+#define fetch_or(ptr, val) \
-+({ typeof(*(ptr)) __old, __val = *(ptr); \
-+ for (;;) { \
-+ __old = cmpxchg((ptr), __val, __val | (val)); \
-+ if (__old == __val) \
-+ break; \
-+ __val = __old; \
-+ } \
-+ __old; \
-+})
-+
-+#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
-+/*
-+ * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
-+ * this avoids any races wrt polling state changes and thereby avoids
-+ * spurious IPIs.
-+ */
-+static bool set_nr_and_not_polling(struct task_struct *p)
-+{
-+ struct thread_info *ti = task_thread_info(p);
-+ return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
-+}
-+
-+/*
-+ * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
-+ *
-+ * If this returns true, then the idle task promises to call
-+ * sched_ttwu_pending() and reschedule soon.
-+ */
-+static bool set_nr_if_polling(struct task_struct *p)
-+{
-+ struct thread_info *ti = task_thread_info(p);
-+ typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags);
-+
-+ for (;;) {
-+ if (!(val & _TIF_POLLING_NRFLAG))
-+ return false;
-+ if (val & _TIF_NEED_RESCHED)
-+ return true;
-+ old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
-+ if (old == val)
-+ break;
-+ val = old;
-+ }
-+ return true;
-+}
-+
-+#else
-+static bool set_nr_and_not_polling(struct task_struct *p)
-+{
-+ set_tsk_need_resched(p);
-+ return true;
-+}
-+
-+#ifdef CONFIG_SMP
-+static bool set_nr_if_polling(struct task_struct *p)
-+{
-+ return false;
-+}
-+#endif
-+#endif
-+
-+/*
-+ * resched_curr - mark rq's current task 'to be rescheduled now'.
-+ *
-+ * On UP this means the setting of the need_resched flag, on SMP it
-+ * might also involve a cross-CPU call to trigger the scheduler on
-+ * the target CPU.
-+ */
-+void resched_curr(struct rq *rq)
-+{
-+ struct task_struct *curr = rq->curr;
-+ int cpu;
-+
-+ lockdep_assert_held(&rq->lock);
-+
-+ if (test_tsk_need_resched(curr))
-+ return;
-+
-+ cpu = cpu_of(rq);
-+
-+ if (cpu == smp_processor_id()) {
-+ set_tsk_need_resched(curr);
-+ set_preempt_need_resched();
-+ return;
-+ }
-+
-+ if (set_nr_and_not_polling(curr))
-+ smp_send_reschedule(cpu);
-+ else
-+ trace_sched_wake_idle_without_ipi(cpu);
-+}
-+
-+void resched_cpu(int cpu)
-+{
-+ struct rq *rq = cpu_rq(cpu);
-+ unsigned long flags;
-+
-+ if (!raw_spin_trylock_irqsave(&rq->lock, flags))
-+ return;
-+ resched_curr(rq);
-+ raw_spin_unlock_irqrestore(&rq->lock, flags);
-+}
-+
-+#ifdef CONFIG_SMP
-+#ifdef CONFIG_NO_HZ_COMMON
-+/*
-+ * In the semi idle case, use the nearest busy cpu for migrating timers
-+ * from an idle cpu. This is good for power-savings.
-+ *
-+ * We don't do similar optimization for completely idle system, as
-+ * selecting an idle cpu will add more delays to the timers than intended
-+ * (as that cpu's timer base may not be uptodate wrt jiffies etc).
-+ */
-+int get_nohz_timer_target(int pinned)
-+{
-+ int cpu = smp_processor_id();
-+ int i;
-+ struct sched_domain *sd;
-+
-+ if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu))
-+ return cpu;
-+
-+ rcu_read_lock();
-+ for_each_domain(cpu, sd) {
-+ for_each_cpu(i, sched_domain_span(sd)) {
-+ if (!idle_cpu(i)) {
-+ cpu = i;
-+ goto unlock;
-+ }
-+ }
-+ }
-+unlock:
-+ rcu_read_unlock();
-+ return cpu;
-+}
-+/*
-+ * When add_timer_on() enqueues a timer into the timer wheel of an
-+ * idle CPU then this timer might expire before the next timer event
-+ * which is scheduled to wake up that CPU. In case of a completely
-+ * idle system the next event might even be infinite time into the
-+ * future. wake_up_idle_cpu() ensures that the CPU is woken up and
-+ * leaves the inner idle loop so the newly added timer is taken into
-+ * account when the CPU goes back to idle and evaluates the timer
-+ * wheel for the next timer event.
-+ */
-+static void wake_up_idle_cpu(int cpu)
-+{
-+ struct rq *rq = cpu_rq(cpu);
-+
-+ if (cpu == smp_processor_id())
-+ return;
-+
-+ if (set_nr_and_not_polling(rq->idle))
-+ smp_send_reschedule(cpu);
-+ else
-+ trace_sched_wake_idle_without_ipi(cpu);
-+}
-+
-+static bool wake_up_full_nohz_cpu(int cpu)
-+{
-+ /*
-+ * We just need the target to call irq_exit() and re-evaluate
-+ * the next tick. The nohz full kick at least implies that.
-+ * If needed we can still optimize that later with an
-+ * empty IRQ.
-+ */
-+ if (tick_nohz_full_cpu(cpu)) {
-+ if (cpu != smp_processor_id() ||
-+ tick_nohz_tick_stopped())
-+ tick_nohz_full_kick_cpu(cpu);
-+ return true;
-+ }
-+
-+ return false;
-+}
-+
-+void wake_up_nohz_cpu(int cpu)
-+{
-+ if (!wake_up_full_nohz_cpu(cpu))
-+ wake_up_idle_cpu(cpu);
-+}
-+
-+static inline bool got_nohz_idle_kick(void)
-+{
-+ int cpu = smp_processor_id();
-+
-+ if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))
-+ return false;
-+
-+ if (idle_cpu(cpu) && !need_resched())
-+ return true;
-+
-+ /*
-+ * We can't run Idle Load Balance on this CPU for this time so we
-+ * cancel it and clear NOHZ_BALANCE_KICK
-+ */
-+ clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
-+ return false;
-+}
-+
-+#else /* CONFIG_NO_HZ_COMMON */
-+
-+static inline bool got_nohz_idle_kick(void)
-+{
-+ return false;
-+}
-+
-+#endif /* CONFIG_NO_HZ_COMMON */
-+
-+#ifdef CONFIG_NO_HZ_FULL
-+bool sched_can_stop_tick(void)
-+{
-+ /*
-+ * FIFO realtime policy runs the highest priority task. Other runnable
-+ * tasks are of a lower priority. The scheduler tick does nothing.
-+ */
-+ if (current->policy == SCHED_FIFO)
-+ return true;
-+
-+ /*
-+ * Round-robin realtime tasks time slice with other tasks at the same
-+ * realtime priority. Is this task the only one at this priority?
-+ */
-+ if (current->policy == SCHED_RR) {
-+ struct sched_rt_entity *rt_se = &current->rt;
-+
-+ return rt_se->run_list.prev == rt_se->run_list.next;
-+ }
-+
-+ /*
-+ * More than one running task need preemption.
-+ * nr_running update is assumed to be visible
-+ * after IPI is sent from wakers.
-+ */
-+ if (this_rq()->nr_running > 1)
-+ return false;
-+
-+ return true;
-+}
-+#endif /* CONFIG_NO_HZ_FULL */
-+
-+void sched_avg_update(struct rq *rq)
-+{
-+ s64 period = sched_avg_period();
-+
-+ while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
-+ /*
-+ * Inline assembly required to prevent the compiler
-+ * optimising this loop into a divmod call.
-+ * See __iter_div_u64_rem() for another example of this.
-+ */
-+ asm("" : "+rm" (rq->age_stamp));
-+ rq->age_stamp += period;
-+ rq->rt_avg /= 2;
-+ }
-+}
-+
-+#endif /* CONFIG_SMP */
-+
-+#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
-+ (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
-+/*
-+ * Iterate task_group tree rooted at *from, calling @down when first entering a
-+ * node and @up when leaving it for the final time.
-+ *
-+ * Caller must hold rcu_lock or sufficient equivalent.
-+ */
-+int walk_tg_tree_from(struct task_group *from,
-+ tg_visitor down, tg_visitor up, void *data)
-+{
-+ struct task_group *parent, *child;
-+ int ret;
-+
-+ parent = from;
-+
-+down:
-+ ret = (*down)(parent, data);
-+ if (ret)
-+ goto out;
-+ list_for_each_entry_rcu(child, &parent->children, siblings) {
-+ parent = child;
-+ goto down;
-+
-+up:
-+ continue;
-+ }
-+ ret = (*up)(parent, data);
-+ if (ret || parent == from)
-+ goto out;
-+
-+ child = parent;
-+ parent = parent->parent;
-+ if (parent)
-+ goto up;
-+out:
-+ return ret;
-+}
-+
-+int tg_nop(struct task_group *tg, void *data)
-+{
-+ return 0;
-+}
-+#endif
-+
-+static void set_load_weight(struct task_struct *p)
-+{
-+ int prio = p->static_prio - MAX_RT_PRIO;
-+ struct load_weight *load = &p->se.load;
-+
-+ /*
-+ * SCHED_IDLE tasks get minimal weight:
-+ */
-+ if (p->policy == SCHED_IDLE) {
-+ load->weight = scale_load(WEIGHT_IDLEPRIO);
-+ load->inv_weight = WMULT_IDLEPRIO;
-+ return;
-+ }
-+
-+ load->weight = scale_load(prio_to_weight[prio]);
-+ load->inv_weight = prio_to_wmult[prio];
-+}
-+
-+static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
-+{
-+ update_rq_clock(rq);
-+ sched_info_queued(rq, p);
-+ p->sched_class->enqueue_task(rq, p, flags);
-+}
-+
-+static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
-+{
-+ update_rq_clock(rq);
-+ sched_info_dequeued(rq, p);
-+ p->sched_class->dequeue_task(rq, p, flags);
-+}
-+
-+void activate_task(struct rq *rq, struct task_struct *p, int flags)
-+{
-+ if (task_contributes_to_load(p))
-+ rq->nr_uninterruptible--;
-+
-+ enqueue_task(rq, p, flags);
-+}
-+
-+void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
-+{
-+ if (task_contributes_to_load(p))
-+ rq->nr_uninterruptible++;
-+
-+ dequeue_task(rq, p, flags);
-+}
-+
-+static void update_rq_clock_task(struct rq *rq, s64 delta)
-+{
-+/*
-+ * In theory, the compile should just see 0 here, and optimize out the call
-+ * to sched_rt_avg_update. But I don't trust it...
-+ */
-+#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
-+ s64 steal = 0, irq_delta = 0;
-+#endif
-+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
-+ irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
-+
-+ /*
-+ * Since irq_time is only updated on {soft,}irq_exit, we might run into
-+ * this case when a previous update_rq_clock() happened inside a
-+ * {soft,}irq region.
-+ *
-+ * When this happens, we stop ->clock_task and only update the
-+ * prev_irq_time stamp to account for the part that fit, so that a next
-+ * update will consume the rest. This ensures ->clock_task is
-+ * monotonic.
-+ *
-+ * It does however cause some slight miss-attribution of {soft,}irq
-+ * time, a more accurate solution would be to update the irq_time using
-+ * the current rq->clock timestamp, except that would require using
-+ * atomic ops.
-+ */
-+ if (irq_delta > delta)
-+ irq_delta = delta;
-+
-+ rq->prev_irq_time += irq_delta;
-+ delta -= irq_delta;
-+#endif
-+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
-+ if (static_key_false((&paravirt_steal_rq_enabled))) {
-+ steal = paravirt_steal_clock(cpu_of(rq));
-+ steal -= rq->prev_steal_time_rq;
-+
-+ if (unlikely(steal > delta))
-+ steal = delta;
-+
-+ rq->prev_steal_time_rq += steal;
-+ delta -= steal;
-+ }
-+#endif
-+
-+ rq->clock_task += delta;
-+
-+#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
-+ if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
-+ sched_rt_avg_update(rq, irq_delta + steal);
-+#endif
-+}
-+
-+void sched_set_stop_task(int cpu, struct task_struct *stop)
-+{
-+ struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
-+ struct task_struct *old_stop = cpu_rq(cpu)->stop;
-+
-+ if (stop) {
-+ /*
-+ * Make it appear like a SCHED_FIFO task, its something
-+ * userspace knows about and won't get confused about.
-+ *
-+ * Also, it will make PI more or less work without too
-+ * much confusion -- but then, stop work should not
-+ * rely on PI working anyway.
-+ */
-+ sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
-+
-+ stop->sched_class = &stop_sched_class;
-+ }
-+
-+ cpu_rq(cpu)->stop = stop;
-+
-+ if (old_stop) {
-+ /*
-+ * Reset it back to a normal scheduling class so that
-+ * it can die in pieces.
-+ */
-+ old_stop->sched_class = &rt_sched_class;
-+ }
-+}
-+
-+/*
-+ * __normal_prio - return the priority that is based on the static prio
-+ */
-+static inline int __normal_prio(struct task_struct *p)
-+{
-+ return p->static_prio;
-+}
-+
-+/*
-+ * Calculate the expected normal priority: i.e. priority
-+ * without taking RT-inheritance into account. Might be
-+ * boosted by interactivity modifiers. Changes upon fork,
-+ * setprio syscalls, and whenever the interactivity
-+ * estimator recalculates.
-+ */
-+static inline int normal_prio(struct task_struct *p)
-+{
-+ int prio;
-+
-+ if (task_has_dl_policy(p))
-+ prio = MAX_DL_PRIO-1;
-+ else if (task_has_rt_policy(p))
-+ prio = MAX_RT_PRIO-1 - p->rt_priority;
-+ else
-+ prio = __normal_prio(p);
-+ return prio;
-+}
-+
-+/*
-+ * Calculate the current priority, i.e. the priority
-+ * taken into account by the scheduler. This value might
-+ * be boosted by RT tasks, or might be boosted by
-+ * interactivity modifiers. Will be RT if the task got
-+ * RT-boosted. If not then it returns p->normal_prio.
-+ */
-+static int effective_prio(struct task_struct *p)
-+{
-+ p->normal_prio = normal_prio(p);
-+ /*
-+ * If we are RT tasks or we were boosted to RT priority,
-+ * keep the priority unchanged. Otherwise, update priority
-+ * to the normal priority:
-+ */
-+ if (!rt_prio(p->prio))
-+ return p->normal_prio;
-+ return p->prio;
-+}
-+
-+/**
-+ * task_curr - is this task currently executing on a CPU?
-+ * @p: the task in question.
-+ *
-+ * Return: 1 if the task is currently executing. 0 otherwise.
-+ */
-+inline int task_curr(const struct task_struct *p)
-+{
-+ return cpu_curr(task_cpu(p)) == p;
-+}
-+
-+/*
-+ * Can drop rq->lock because from sched_class::switched_from() methods drop it.
-+ */
-+static inline void check_class_changed(struct rq *rq, struct task_struct *p,
-+ const struct sched_class *prev_class,
-+ int oldprio)
-+{
-+ if (prev_class != p->sched_class) {
-+ if (prev_class->switched_from)
-+ prev_class->switched_from(rq, p);
-+ /* Possble rq->lock 'hole'. */
-+ p->sched_class->switched_to(rq, p);
-+ } else if (oldprio != p->prio || dl_task(p))
-+ p->sched_class->prio_changed(rq, p, oldprio);
-+}
-+
-+void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
-+{
-+ const struct sched_class *class;
-+
-+ if (p->sched_class == rq->curr->sched_class) {
-+ rq->curr->sched_class->check_preempt_curr(rq, p, flags);
-+ } else {
-+ for_each_class(class) {
-+ if (class == rq->curr->sched_class)
-+ break;
-+ if (class == p->sched_class) {
-+ resched_curr(rq);
-+ break;
-+ }
-+ }
-+ }
-+
-+ /*
-+ * A queue event has occurred, and we're going to schedule. In
-+ * this case, we can save a useless back to back clock update.
-+ */
-+ if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
-+ rq_clock_skip_update(rq, true);
-+}
-+
-+#ifdef CONFIG_SMP
-+void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
-+{
-+#ifdef CONFIG_SCHED_DEBUG
-+ /*
-+ * We should never call set_task_cpu() on a blocked task,
-+ * ttwu() will sort out the placement.
-+ */
-+ WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
-+ !p->on_rq);
-+
-+#ifdef CONFIG_LOCKDEP
-+ /*
-+ * The caller should hold either p->pi_lock or rq->lock, when changing
-+ * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
-+ *
-+ * sched_move_task() holds both and thus holding either pins the cgroup,
-+ * see task_group().
-+ *
-+ * Furthermore, all task_rq users should acquire both locks, see
-+ * task_rq_lock().
-+ */
-+ WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
-+ lockdep_is_held(&task_rq(p)->lock)));
-+#endif
-+#endif
-+
-+ trace_sched_migrate_task(p, new_cpu);
-+
-+ if (task_cpu(p) != new_cpu) {
-+ if (p->sched_class->migrate_task_rq)
-+ p->sched_class->migrate_task_rq(p, new_cpu);
-+ p->se.nr_migrations++;
-+ perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0);
-+ }
-+
-+ __set_task_cpu(p, new_cpu);
-+}
-+
-+static void __migrate_swap_task(struct task_struct *p, int cpu)
-+{
-+ if (task_on_rq_queued(p)) {
-+ struct rq *src_rq, *dst_rq;
-+
-+ src_rq = task_rq(p);
-+ dst_rq = cpu_rq(cpu);
-+
-+ deactivate_task(src_rq, p, 0);
-+ set_task_cpu(p, cpu);
-+ activate_task(dst_rq, p, 0);
-+ check_preempt_curr(dst_rq, p, 0);
-+ } else {
-+ /*
-+ * Task isn't running anymore; make it appear like we migrated
-+ * it before it went to sleep. This means on wakeup we make the
-+ * previous cpu our targer instead of where it really is.
-+ */
-+ p->wake_cpu = cpu;
-+ }
-+}
-+
-+struct migration_swap_arg {
-+ struct task_struct *src_task, *dst_task;
-+ int src_cpu, dst_cpu;
-+};
-+
-+static int migrate_swap_stop(void *data)
-+{
-+ struct migration_swap_arg *arg = data;
-+ struct rq *src_rq, *dst_rq;
-+ int ret = -EAGAIN;
-+
-+ src_rq = cpu_rq(arg->src_cpu);
-+ dst_rq = cpu_rq(arg->dst_cpu);
-+
-+ double_raw_lock(&arg->src_task->pi_lock,
-+ &arg->dst_task->pi_lock);
-+ double_rq_lock(src_rq, dst_rq);
-+ if (task_cpu(arg->dst_task) != arg->dst_cpu)
-+ goto unlock;
-+
-+ if (task_cpu(arg->src_task) != arg->src_cpu)
-+ goto unlock;
-+
-+ if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
-+ goto unlock;
-+
-+ if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
-+ goto unlock;
-+
-+ __migrate_swap_task(arg->src_task, arg->dst_cpu);
-+ __migrate_swap_task(arg->dst_task, arg->src_cpu);
-+
-+ ret = 0;
-+
-+unlock:
-+ double_rq_unlock(src_rq, dst_rq);
-+ raw_spin_unlock(&arg->dst_task->pi_lock);
-+ raw_spin_unlock(&arg->src_task->pi_lock);
-+
-+ return ret;
-+}
-+
-+/*
-+ * Cross migrate two tasks
-+ */
-+int migrate_swap(struct task_struct *cur, struct task_struct *p)
-+{
-+ struct migration_swap_arg arg;
-+ int ret = -EINVAL;
-+
-+ arg = (struct migration_swap_arg){
-+ .src_task = cur,
-+ .src_cpu = task_cpu(cur),
-+ .dst_task = p,
-+ .dst_cpu = task_cpu(p),
-+ };
-+
-+ if (arg.src_cpu == arg.dst_cpu)
-+ goto out;
-+
-+ /*
-+ * These three tests are all lockless; this is OK since all of them
-+ * will be re-checked with proper locks held further down the line.
-+ */
-+ if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
-+ goto out;
-+
-+ if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
-+ goto out;
-+
-+ if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
-+ goto out;
-+
-+ trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
-+ ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
-+
-+out:
-+ return ret;
-+}
-+
-+struct migration_arg {
-+ struct task_struct *task;
-+ int dest_cpu;
-+};
-+
-+static int migration_cpu_stop(void *data);
-+
-+/*
-+ * wait_task_inactive - wait for a thread to unschedule.
-+ *
-+ * If @match_state is nonzero, it's the @p->state value just checked and
-+ * not expected to change. If it changes, i.e. @p might have woken up,
-+ * then return zero. When we succeed in waiting for @p to be off its CPU,
-+ * we return a positive number (its total switch count). If a second call
-+ * a short while later returns the same number, the caller can be sure that
-+ * @p has remained unscheduled the whole time.
-+ *
-+ * The caller must ensure that the task *will* unschedule sometime soon,
-+ * else this function might spin for a *long* time. This function can't
-+ * be called with interrupts off, or it may introduce deadlock with
-+ * smp_call_function() if an IPI is sent by the same process we are
-+ * waiting to become inactive.
-+ */
-+unsigned long wait_task_inactive(struct task_struct *p, long match_state)
-+{
-+ unsigned long flags;
-+ int running, queued;
-+ unsigned long ncsw;
-+ struct rq *rq;
-+
-+ for (;;) {
-+ /*
-+ * We do the initial early heuristics without holding
-+ * any task-queue locks at all. We'll only try to get
-+ * the runqueue lock when things look like they will
-+ * work out!
-+ */
-+ rq = task_rq(p);
-+
-+ /*
-+ * If the task is actively running on another CPU
-+ * still, just relax and busy-wait without holding
-+ * any locks.
-+ *
-+ * NOTE! Since we don't hold any locks, it's not
-+ * even sure that "rq" stays as the right runqueue!
-+ * But we don't care, since "task_running()" will
-+ * return false if the runqueue has changed and p
-+ * is actually now running somewhere else!
-+ */
-+ while (task_running(rq, p)) {
-+ if (match_state && unlikely(p->state != match_state))
-+ return 0;
-+ cpu_relax();
-+ }
-+
-+ /*
-+ * Ok, time to look more closely! We need the rq
-+ * lock now, to be *sure*. If we're wrong, we'll
-+ * just go back and repeat.
-+ */
-+ rq = task_rq_lock(p, &flags);
-+ trace_sched_wait_task(p);
-+ running = task_running(rq, p);
-+ queued = task_on_rq_queued(p);
-+ ncsw = 0;
-+ if (!match_state || p->state == match_state)
-+ ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
-+ task_rq_unlock(rq, p, &flags);
-+
-+ /*
-+ * If it changed from the expected state, bail out now.
-+ */
-+ if (unlikely(!ncsw))
-+ break;
-+
-+ /*
-+ * Was it really running after all now that we
-+ * checked with the proper locks actually held?
-+ *
-+ * Oops. Go back and try again..
-+ */
-+ if (unlikely(running)) {
-+ cpu_relax();
-+ continue;
-+ }
-+
-+ /*
-+ * It's not enough that it's not actively running,
-+ * it must be off the runqueue _entirely_, and not
-+ * preempted!
-+ *
-+ * So if it was still runnable (but just not actively
-+ * running right now), it's preempted, and we should
-+ * yield - it could be a while.
-+ */
-+ if (unlikely(queued)) {
-+ ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);
-+
-+ set_current_state(TASK_UNINTERRUPTIBLE);
-+ schedule_hrtimeout(&to, HRTIMER_MODE_REL);
-+ continue;
-+ }
-+
-+ /*
-+ * Ahh, all good. It wasn't running, and it wasn't
-+ * runnable, which means that it will never become
-+ * running in the future either. We're all done!
-+ */
-+ break;
-+ }
-+
-+ return ncsw;
-+}
-+
-+/***
-+ * kick_process - kick a running thread to enter/exit the kernel
-+ * @p: the to-be-kicked thread
-+ *
-+ * Cause a process which is running on another CPU to enter
-+ * kernel-mode, without any delay. (to get signals handled.)
-+ *
-+ * NOTE: this function doesn't have to take the runqueue lock,
-+ * because all it wants to ensure is that the remote task enters
-+ * the kernel. If the IPI races and the task has been migrated
-+ * to another CPU then no harm is done and the purpose has been
-+ * achieved as well.
-+ */
-+void kick_process(struct task_struct *p)
-+{
-+ int cpu;
-+
-+ preempt_disable();
-+ cpu = task_cpu(p);
-+ if ((cpu != smp_processor_id()) && task_curr(p))
-+ smp_send_reschedule(cpu);
-+ preempt_enable();
-+}
-+EXPORT_SYMBOL_GPL(kick_process);
-+#endif /* CONFIG_SMP */
-+
-+#ifdef CONFIG_SMP
-+/*
-+ * ->cpus_allowed is protected by both rq->lock and p->pi_lock
-+ */
-+static int select_fallback_rq(int cpu, struct task_struct *p)
-+{
-+ int nid = cpu_to_node(cpu);
-+ const struct cpumask *nodemask = NULL;
-+ enum { cpuset, possible, fail } state = cpuset;
-+ int dest_cpu;
-+
-+ /*
-+ * If the node that the cpu is on has been offlined, cpu_to_node()
-+ * will return -1. There is no cpu on the node, and we should
-+ * select the cpu on the other node.
-+ */
-+ if (nid != -1) {
-+ nodemask = cpumask_of_node(nid);
-+
-+ /* Look for allowed, online CPU in same node. */
-+ for_each_cpu(dest_cpu, nodemask) {
-+ if (!cpu_online(dest_cpu))
-+ continue;
-+ if (!cpu_active(dest_cpu))
-+ continue;
-+ if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
-+ return dest_cpu;
-+ }
-+ }
-+
-+ for (;;) {
-+ /* Any allowed, online CPU? */
-+ for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
-+ if (!cpu_online(dest_cpu))
-+ continue;
-+ if (!cpu_active(dest_cpu))
-+ continue;
-+ goto out;
-+ }
-+
-+ switch (state) {
-+ case cpuset:
-+ /* No more Mr. Nice Guy. */
-+ cpuset_cpus_allowed_fallback(p);
-+ state = possible;
-+ break;
-+
-+ case possible:
-+ do_set_cpus_allowed(p, cpu_possible_mask);
-+ state = fail;
-+ break;
-+
-+ case fail:
-+ BUG();
-+ break;
-+ }
-+ }
-+
-+out:
-+ if (state != cpuset) {
-+ /*
-+ * Don't tell them about moving exiting tasks or
-+ * kernel threads (both mm NULL), since they never
-+ * leave kernel.
-+ */
-+ if (p->mm && printk_ratelimit()) {
-+ printk_deferred("process %d (%s) no longer affine to cpu%d\n",
-+ task_pid_nr(p), p->comm, cpu);
-+ }
-+ }
-+
-+ return dest_cpu;
-+}
-+
-+/*
-+ * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
-+ */
-+static inline
-+int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
-+{
-+ if (p->nr_cpus_allowed > 1)
-+ cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
-+
-+ /*
-+ * In order not to call set_task_cpu() on a blocking task we need
-+ * to rely on ttwu() to place the task on a valid ->cpus_allowed
-+ * cpu.
-+ *
-+ * Since this is common to all placement strategies, this lives here.
-+ *
-+ * [ this allows ->select_task() to simply return task_cpu(p) and
-+ * not worry about this generic constraint ]
-+ */
-+ if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
-+ !cpu_online(cpu)))
-+ cpu = select_fallback_rq(task_cpu(p), p);
-+
-+ return cpu;
-+}
-+
-+static void update_avg(u64 *avg, u64 sample)
-+{
-+ s64 diff = sample - *avg;
-+ *avg += diff >> 3;
-+}
-+#endif
-+
-+static void
-+ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
-+{
-+#ifdef CONFIG_SCHEDSTATS
-+ struct rq *rq = this_rq();
-+
-+#ifdef CONFIG_SMP
-+ int this_cpu = smp_processor_id();
-+
-+ if (cpu == this_cpu) {
-+ schedstat_inc(rq, ttwu_local);
-+ schedstat_inc(p, se.statistics.nr_wakeups_local);
-+ } else {
-+ struct sched_domain *sd;
-+
-+ schedstat_inc(p, se.statistics.nr_wakeups_remote);
-+ rcu_read_lock();
-+ for_each_domain(this_cpu, sd) {
-+ if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
-+ schedstat_inc(sd, ttwu_wake_remote);
-+ break;
-+ }
-+ }
-+ rcu_read_unlock();
-+ }
-+
-+ if (wake_flags & WF_MIGRATED)
-+ schedstat_inc(p, se.statistics.nr_wakeups_migrate);
-+
-+#endif /* CONFIG_SMP */
-+
-+ schedstat_inc(rq, ttwu_count);
-+ schedstat_inc(p, se.statistics.nr_wakeups);
-+
-+ if (wake_flags & WF_SYNC)
-+ schedstat_inc(p, se.statistics.nr_wakeups_sync);
-+
-+#endif /* CONFIG_SCHEDSTATS */
-+}
-+
-+static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
-+{
-+ activate_task(rq, p, en_flags);
-+ p->on_rq = TASK_ON_RQ_QUEUED;
-+
-+ /* if a worker is waking up, notify workqueue */
-+ if (p->flags & PF_WQ_WORKER)
-+ wq_worker_waking_up(p, cpu_of(rq));
-+}
-+
-+/*
-+ * Mark the task runnable and perform wakeup-preemption.
-+ */
-+static void
-+ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
-+{
-+ check_preempt_curr(rq, p, wake_flags);
-+ trace_sched_wakeup(p, true);
-+
-+ p->state = TASK_RUNNING;
-+#ifdef CONFIG_SMP
-+ if (p->sched_class->task_woken)
-+ p->sched_class->task_woken(rq, p);
-+
-+ if (rq->idle_stamp) {
-+ u64 delta = rq_clock(rq) - rq->idle_stamp;
-+ u64 max = 2*rq->max_idle_balance_cost;
-+
-+ update_avg(&rq->avg_idle, delta);
-+
-+ if (rq->avg_idle > max)
-+ rq->avg_idle = max;
-+
-+ rq->idle_stamp = 0;
-+ }
-+#endif
-+}
-+
-+static void
-+ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
-+{
-+#ifdef CONFIG_SMP
-+ if (p->sched_contributes_to_load)
-+ rq->nr_uninterruptible--;
-+#endif
-+
-+ ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
-+ ttwu_do_wakeup(rq, p, wake_flags);
-+}
-+
-+/*
-+ * Called in case the task @p isn't fully descheduled from its runqueue,
-+ * in this case we must do a remote wakeup. Its a 'light' wakeup though,
-+ * since all we need to do is flip p->state to TASK_RUNNING, since
-+ * the task is still ->on_rq.
-+ */
-+static int ttwu_remote(struct task_struct *p, int wake_flags)
-+{
-+ struct rq *rq;
-+ int ret = 0;
-+
-+ rq = __task_rq_lock(p);
-+ if (task_on_rq_queued(p)) {
-+ /* check_preempt_curr() may use rq clock */
-+ update_rq_clock(rq);
-+ ttwu_do_wakeup(rq, p, wake_flags);
-+ ret = 1;
-+ }
-+ __task_rq_unlock(rq);
-+
-+ return ret;
-+}
-+
-+#ifdef CONFIG_SMP
-+void sched_ttwu_pending(void)
-+{
-+ struct rq *rq = this_rq();
-+ struct llist_node *llist = llist_del_all(&rq->wake_list);
-+ struct task_struct *p;
-+ unsigned long flags;
-+
-+ if (!llist)
-+ return;
-+
-+ raw_spin_lock_irqsave(&rq->lock, flags);
-+
-+ while (llist) {
-+ p = llist_entry(llist, struct task_struct, wake_entry);
-+ llist = llist_next(llist);
-+ ttwu_do_activate(rq, p, 0);
-+ }
-+
-+ raw_spin_unlock_irqrestore(&rq->lock, flags);
-+}
-+
-+void scheduler_ipi(void)
-+{
-+ /*
-+ * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
-+ * TIF_NEED_RESCHED remotely (for the first time) will also send
-+ * this IPI.
-+ */
-+ preempt_fold_need_resched();
-+
-+ if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
-+ return;
-+
-+ /*
-+ * Not all reschedule IPI handlers call irq_enter/irq_exit, since
-+ * traditionally all their work was done from the interrupt return
-+ * path. Now that we actually do some work, we need to make sure
-+ * we do call them.
-+ *
-+ * Some archs already do call them, luckily irq_enter/exit nest
-+ * properly.
-+ *
-+ * Arguably we should visit all archs and update all handlers,
-+ * however a fair share of IPIs are still resched only so this would
-+ * somewhat pessimize the simple resched case.
-+ */
-+ irq_enter();
-+ sched_ttwu_pending();
-+
-+ /*
-+ * Check if someone kicked us for doing the nohz idle load balance.
-+ */
-+ if (unlikely(got_nohz_idle_kick())) {
-+ this_rq()->idle_balance = 1;
-+ raise_softirq_irqoff(SCHED_SOFTIRQ);
-+ }
-+ irq_exit();
-+}
-+
-+static void ttwu_queue_remote(struct task_struct *p, int cpu)
-+{
-+ struct rq *rq = cpu_rq(cpu);
-+
-+ if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
-+ if (!set_nr_if_polling(rq->idle))
-+ smp_send_reschedule(cpu);
-+ else
-+ trace_sched_wake_idle_without_ipi(cpu);
-+ }
-+}
-+
-+void wake_up_if_idle(int cpu)
-+{
-+ struct rq *rq = cpu_rq(cpu);
-+ unsigned long flags;
-+
-+ rcu_read_lock();
-+
-+ if (!is_idle_task(rcu_dereference(rq->curr)))
-+ goto out;
-+
-+ if (set_nr_if_polling(rq->idle)) {
-+ trace_sched_wake_idle_without_ipi(cpu);
-+ } else {
-+ raw_spin_lock_irqsave(&rq->lock, flags);
-+ if (is_idle_task(rq->curr))
-+ smp_send_reschedule(cpu);
-+ /* Else cpu is not in idle, do nothing here */
-+ raw_spin_unlock_irqrestore(&rq->lock, flags);
-+ }
-+
-+out:
-+ rcu_read_unlock();
-+}
-+
-+bool cpus_share_cache(int this_cpu, int that_cpu)
-+{
-+ return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
-+}
-+#endif /* CONFIG_SMP */
-+
-+static void ttwu_queue(struct task_struct *p, int cpu)
-+{
-+ struct rq *rq = cpu_rq(cpu);
-+
-+#if defined(CONFIG_SMP)
-+ if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
-+ sched_clock_cpu(cpu); /* sync clocks x-cpu */
-+ ttwu_queue_remote(p, cpu);
-+ return;
-+ }
-+#endif
-+
-+ raw_spin_lock(&rq->lock);
-+ ttwu_do_activate(rq, p, 0);
-+ raw_spin_unlock(&rq->lock);
-+}
-+
-+/**
-+ * try_to_wake_up - wake up a thread
-+ * @p: the thread to be awakened
-+ * @state: the mask of task states that can be woken
-+ * @wake_flags: wake modifier flags (WF_*)
-+ *
-+ * Put it on the run-queue if it's not already there. The "current"
-+ * thread is always on the run-queue (except when the actual
-+ * re-schedule is in progress), and as such you're allowed to do
-+ * the simpler "current->state = TASK_RUNNING" to mark yourself
-+ * runnable without the overhead of this.
-+ *
-+ * Return: %true if @p was woken up, %false if it was already running.
-+ * or @state didn't match @p's state.
-+ */
-+static int
-+try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
-+{
-+ unsigned long flags;
-+ int cpu, success = 0;
-+
-+ /*
-+ * If we are going to wake up a thread waiting for CONDITION we
-+ * need to ensure that CONDITION=1 done by the caller can not be
-+ * reordered with p->state check below. This pairs with mb() in
-+ * set_current_state() the waiting thread does.
-+ */
-+ smp_mb__before_spinlock();
-+ raw_spin_lock_irqsave(&p->pi_lock, flags);
-+ if (!(p->state & state))
-+ goto out;
-+
-+ success = 1; /* we're going to change ->state */
-+ cpu = task_cpu(p);
-+
-+ if (p->on_rq && ttwu_remote(p, wake_flags))
-+ goto stat;
-+
-+#ifdef CONFIG_SMP
-+ /*
-+ * If the owning (remote) cpu is still in the middle of schedule() with
-+ * this task as prev, wait until its done referencing the task.
-+ */
-+ while (p->on_cpu)
-+ cpu_relax();
-+ /*
-+ * Pairs with the smp_wmb() in finish_lock_switch().
-+ */
-+ smp_rmb();
-+
-+ p->sched_contributes_to_load = !!task_contributes_to_load(p);
-+ p->state = TASK_WAKING;
-+
-+ if (p->sched_class->task_waking)
-+ p->sched_class->task_waking(p);
-+
-+ cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
-+ if (task_cpu(p) != cpu) {
-+ wake_flags |= WF_MIGRATED;
-+ set_task_cpu(p, cpu);
-+ }
-+#endif /* CONFIG_SMP */
-+
-+ ttwu_queue(p, cpu);
-+stat:
-+ ttwu_stat(p, cpu, wake_flags);
-+out:
-+ raw_spin_unlock_irqrestore(&p->pi_lock, flags);
-+
-+ return success;
-+}
-+
-+/**
-+ * try_to_wake_up_local - try to wake up a local task with rq lock held
-+ * @p: the thread to be awakened
-+ *
-+ * Put @p on the run-queue if it's not already there. The caller must
-+ * ensure that this_rq() is locked, @p is bound to this_rq() and not
-+ * the current task.
-+ */
-+static void try_to_wake_up_local(struct task_struct *p)
-+{
-+ struct rq *rq = task_rq(p);
-+
-+ if (WARN_ON_ONCE(rq != this_rq()) ||
-+ WARN_ON_ONCE(p == current))
-+ return;
-+
-+ lockdep_assert_held(&rq->lock);
-+
-+ if (!raw_spin_trylock(&p->pi_lock)) {
-+ raw_spin_unlock(&rq->lock);
-+ raw_spin_lock(&p->pi_lock);
-+ raw_spin_lock(&rq->lock);
-+ }
-+
-+ if (!(p->state & TASK_NORMAL))
-+ goto out;
-+
-+ if (!task_on_rq_queued(p))
-+ ttwu_activate(rq, p, ENQUEUE_WAKEUP);
-+
-+ ttwu_do_wakeup(rq, p, 0);
-+ ttwu_stat(p, smp_processor_id(), 0);
-+out:
-+ raw_spin_unlock(&p->pi_lock);
-+}
-+
-+/**
-+ * wake_up_process - Wake up a specific process
-+ * @p: The process to be woken up.
-+ *
-+ * Attempt to wake up the nominated process and move it to the set of runnable
-+ * processes.
-+ *
-+ * Return: 1 if the process was woken up, 0 if it was already running.
-+ *
-+ * It may be assumed that this function implies a write memory barrier before
-+ * changing the task state if and only if any tasks are woken up.
-+ */
-+int wake_up_process(struct task_struct *p)
-+{
-+ WARN_ON(task_is_stopped_or_traced(p));
-+ return try_to_wake_up(p, TASK_NORMAL, 0);
-+}
-+EXPORT_SYMBOL(wake_up_process);
-+
-+int wake_up_state(struct task_struct *p, unsigned int state)
-+{
-+ return try_to_wake_up(p, state, 0);
-+}
-+
-+/*
-+ * This function clears the sched_dl_entity static params.
-+ */
-+void __dl_clear_params(struct task_struct *p)
-+{
-+ struct sched_dl_entity *dl_se = &p->dl;
-+
-+ dl_se->dl_runtime = 0;
-+ dl_se->dl_deadline = 0;
-+ dl_se->dl_period = 0;
-+ dl_se->flags = 0;
-+ dl_se->dl_bw = 0;
-+
-+ dl_se->dl_throttled = 0;
-+ dl_se->dl_new = 1;
-+ dl_se->dl_yielded = 0;
-+}
-+
-+/*
-+ * Perform scheduler related setup for a newly forked process p.
-+ * p is forked by current.
-+ *
-+ * __sched_fork() is basic setup used by init_idle() too:
-+ */
-+static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
-+{
-+ p->on_rq = 0;
-+
-+ p->se.on_rq = 0;
-+ p->se.exec_start = 0;
-+ p->se.sum_exec_runtime = 0;
-+ p->se.prev_sum_exec_runtime = 0;
-+ p->se.nr_migrations = 0;
-+ p->se.vruntime = 0;
-+#ifdef CONFIG_SMP
-+ p->se.avg.decay_count = 0;
-+#endif
-+ INIT_LIST_HEAD(&p->se.group_node);
-+
-+#ifdef CONFIG_SCHEDSTATS
-+ memset(&p->se.statistics, 0, sizeof(p->se.statistics));
-+#endif
-+
-+ RB_CLEAR_NODE(&p->dl.rb_node);
-+ init_dl_task_timer(&p->dl);
-+ __dl_clear_params(p);
-+
-+ INIT_LIST_HEAD(&p->rt.run_list);
-+
-+#ifdef CONFIG_PREEMPT_NOTIFIERS
-+ INIT_HLIST_HEAD(&p->preempt_notifiers);
-+#endif
-+
-+#ifdef CONFIG_NUMA_BALANCING
-+ if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
-+ p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
-+ p->mm->numa_scan_seq = 0;
-+ }
-+
-+ if (clone_flags & CLONE_VM)
-+ p->numa_preferred_nid = current->numa_preferred_nid;
-+ else
-+ p->numa_preferred_nid = -1;
-+
-+ p->node_stamp = 0ULL;
-+ p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
-+ p->numa_scan_period = sysctl_numa_balancing_scan_delay;
-+ p->numa_work.next = &p->numa_work;
-+ p->numa_faults = NULL;
-+ p->last_task_numa_placement = 0;
-+ p->last_sum_exec_runtime = 0;
-+
-+ p->numa_group = NULL;
-+#endif /* CONFIG_NUMA_BALANCING */
-+}
-+
-+#ifdef CONFIG_NUMA_BALANCING
-+#ifdef CONFIG_SCHED_DEBUG
-+void set_numabalancing_state(bool enabled)
-+{
-+ if (enabled)
-+ sched_feat_set("NUMA");
-+ else
-+ sched_feat_set("NO_NUMA");
-+}
-+#else
-+__read_mostly bool numabalancing_enabled;
-+
-+void set_numabalancing_state(bool enabled)
-+{
-+ numabalancing_enabled = enabled;
-+}
-+#endif /* CONFIG_SCHED_DEBUG */
-+
-+#ifdef CONFIG_PROC_SYSCTL
-+int sysctl_numa_balancing(struct ctl_table *table, int write,
-+ void __user *buffer, size_t *lenp, loff_t *ppos)
-+{
-+ struct ctl_table t;
-+ int err;
-+ int state = numabalancing_enabled;
-+
-+ if (write && !capable(CAP_SYS_ADMIN))
-+ return -EPERM;
-+
-+ t = *table;
-+ t.data = &state;
-+ err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
-+ if (err < 0)
-+ return err;
-+ if (write)
-+ set_numabalancing_state(state);
-+ return err;
-+}
-+#endif
-+#endif
-+
-+/*
-+ * fork()/clone()-time setup:
-+ */
-+int sched_fork(unsigned long clone_flags, struct task_struct *p)
-+{
-+ unsigned long flags;
-+ int cpu = get_cpu();
-+
-+ __sched_fork(clone_flags, p);
-+ /*
-+ * We mark the process as running here. This guarantees that
-+ * nobody will actually run it, and a signal or other external
-+ * event cannot wake it up and insert it on the runqueue either.
-+ */
-+ p->state = TASK_RUNNING;
-+
-+ /*
-+ * Make sure we do not leak PI boosting priority to the child.
-+ */
-+ p->prio = current->normal_prio;
-+
-+ /*
-+ * Revert to default priority/policy on fork if requested.
-+ */
-+ if (unlikely(p->sched_reset_on_fork)) {
-+ if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
-+ p->policy = SCHED_NORMAL;
-+ p->static_prio = NICE_TO_PRIO(0);
-+ p->rt_priority = 0;
-+ } else if (PRIO_TO_NICE(p->static_prio) < 0)
-+ p->static_prio = NICE_TO_PRIO(0);
-+
-+ p->prio = p->normal_prio = __normal_prio(p);
-+ set_load_weight(p);
-+
-+ /*
-+ * We don't need the reset flag anymore after the fork. It has
-+ * fulfilled its duty:
-+ */
-+ p->sched_reset_on_fork = 0;
-+ }
-+
-+ if (dl_prio(p->prio)) {
-+ put_cpu();
-+ return -EAGAIN;
-+ } else if (rt_prio(p->prio)) {
-+ p->sched_class = &rt_sched_class;
-+ } else {
-+ p->sched_class = &fair_sched_class;
-+ }
-+
-+ if (p->sched_class->task_fork)
-+ p->sched_class->task_fork(p);
-+
-+ /*
-+ * The child is not yet in the pid-hash so no cgroup attach races,
-+ * and the cgroup is pinned to this child due to cgroup_fork()
-+ * is ran before sched_fork().
-+ *
-+ * Silence PROVE_RCU.
-+ */
-+ raw_spin_lock_irqsave(&p->pi_lock, flags);
-+ set_task_cpu(p, cpu);
-+ raw_spin_unlock_irqrestore(&p->pi_lock, flags);
-+
-+#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
-+ if (likely(sched_info_on()))
-+ memset(&p->sched_info, 0, sizeof(p->sched_info));
-+#endif
-+#if defined(CONFIG_SMP)
-+ p->on_cpu = 0;
-+#endif
-+ init_task_preempt_count(p);
-+#ifdef CONFIG_SMP
-+ plist_node_init(&p->pushable_tasks, MAX_PRIO);
-+ RB_CLEAR_NODE(&p->pushable_dl_tasks);
-+#endif
-+
-+ put_cpu();
-+ return 0;
-+}
-+
-+unsigned long to_ratio(u64 period, u64 runtime)
-+{
-+ if (runtime == RUNTIME_INF)
-+ return 1ULL << 20;
-+
-+ /*
-+ * Doing this here saves a lot of checks in all
-+ * the calling paths, and returning zero seems
-+ * safe for them anyway.
-+ */
-+ if (period == 0)
-+ return 0;
-+
-+ return div64_u64(runtime << 20, period);
-+}
-+
-+#ifdef CONFIG_SMP
-+inline struct dl_bw *dl_bw_of(int i)
-+{
-+ rcu_lockdep_assert(rcu_read_lock_sched_held(),
-+ "sched RCU must be held");
-+ return &cpu_rq(i)->rd->dl_bw;
-+}
-+
-+static inline int dl_bw_cpus(int i)
-+{
-+ struct root_domain *rd = cpu_rq(i)->rd;
-+ int cpus = 0;
-+
-+ rcu_lockdep_assert(rcu_read_lock_sched_held(),
-+ "sched RCU must be held");
-+ for_each_cpu_and(i, rd->span, cpu_active_mask)
-+ cpus++;
-+
-+ return cpus;
-+}
-+#else
-+inline struct dl_bw *dl_bw_of(int i)
-+{
-+ return &cpu_rq(i)->dl.dl_bw;
-+}
-+
-+static inline int dl_bw_cpus(int i)
-+{
-+ return 1;
-+}
-+#endif
-+
-+/*
-+ * We must be sure that accepting a new task (or allowing changing the
-+ * parameters of an existing one) is consistent with the bandwidth
-+ * constraints. If yes, this function also accordingly updates the currently
-+ * allocated bandwidth to reflect the new situation.
-+ *
-+ * This function is called while holding p's rq->lock.
-+ *
-+ * XXX we should delay bw change until the task's 0-lag point, see
-+ * __setparam_dl().
-+ */
-+static int dl_overflow(struct task_struct *p, int policy,
-+ const struct sched_attr *attr)
-+{
-+
-+ struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
-+ u64 period = attr->sched_period ?: attr->sched_deadline;
-+ u64 runtime = attr->sched_runtime;
-+ u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
-+ int cpus, err = -1;
-+
-+ if (new_bw == p->dl.dl_bw)
-+ return 0;
-+
-+ /*
-+ * Either if a task, enters, leave, or stays -deadline but changes
-+ * its parameters, we may need to update accordingly the total
-+ * allocated bandwidth of the container.
-+ */
-+ raw_spin_lock(&dl_b->lock);
-+ cpus = dl_bw_cpus(task_cpu(p));
-+ if (dl_policy(policy) && !task_has_dl_policy(p) &&
-+ !__dl_overflow(dl_b, cpus, 0, new_bw)) {
-+ __dl_add(dl_b, new_bw);
-+ err = 0;
-+ } else if (dl_policy(policy) && task_has_dl_policy(p) &&
-+ !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
-+ __dl_clear(dl_b, p->dl.dl_bw);
-+ __dl_add(dl_b, new_bw);
-+ err = 0;
-+ } else if (!dl_policy(policy) && task_has_dl_policy(p)) {
-+ __dl_clear(dl_b, p->dl.dl_bw);
-+ err = 0;
-+ }
-+ raw_spin_unlock(&dl_b->lock);
-+
-+ return err;
-+}
-+
-+extern void init_dl_bw(struct dl_bw *dl_b);
-+
-+/*
-+ * wake_up_new_task - wake up a newly created task for the first time.
-+ *
-+ * This function will do some initial scheduler statistics housekeeping
-+ * that must be done for every newly created context, then puts the task
-+ * on the runqueue and wakes it.
-+ */
-+void wake_up_new_task(struct task_struct *p)
-+{
-+ unsigned long flags;
-+ struct rq *rq;
-+
-+ raw_spin_lock_irqsave(&p->pi_lock, flags);
-+#ifdef CONFIG_SMP
-+ /*
-+ * Fork balancing, do it here and not earlier because:
-+ * - cpus_allowed can change in the fork path
-+ * - any previously selected cpu might disappear through hotplug
-+ */
-+ set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
-+#endif
-+
-+ /* Initialize new task's runnable average */
-+ init_task_runnable_average(p);
-+ rq = __task_rq_lock(p);
-+ activate_task(rq, p, 0);
-+ p->on_rq = TASK_ON_RQ_QUEUED;
-+ trace_sched_wakeup_new(p, true);
-+ check_preempt_curr(rq, p, WF_FORK);
-+#ifdef CONFIG_SMP
-+ if (p->sched_class->task_woken)
-+ p->sched_class->task_woken(rq, p);
-+#endif
-+ task_rq_unlock(rq, p, &flags);
-+}
-+
-+#ifdef CONFIG_PREEMPT_NOTIFIERS
-+
-+/**
-+ * preempt_notifier_register - tell me when current is being preempted & rescheduled
-+ * @notifier: notifier struct to register
-+ */
-+void preempt_notifier_register(struct preempt_notifier *notifier)
-+{
-+ hlist_add_head(&notifier->link, &current->preempt_notifiers);
-+}
-+EXPORT_SYMBOL_GPL(preempt_notifier_register);
-+
-+/**
-+ * preempt_notifier_unregister - no longer interested in preemption notifications
-+ * @notifier: notifier struct to unregister
-+ *
-+ * This is safe to call from within a preemption notifier.
-+ */
-+void preempt_notifier_unregister(struct preempt_notifier *notifier)
-+{
-+ hlist_del(&notifier->link);
-+}
-+EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
-+
-+static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
-+{
-+ struct preempt_notifier *notifier;
-+
-+ hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
-+ notifier->ops->sched_in(notifier, raw_smp_processor_id());
-+}
-+
-+static void
-+fire_sched_out_preempt_notifiers(struct task_struct *curr,
-+ struct task_struct *next)
-+{
-+ struct preempt_notifier *notifier;
-+
-+ hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
-+ notifier->ops->sched_out(notifier, next);
-+}
-+
-+#else /* !CONFIG_PREEMPT_NOTIFIERS */
-+
-+static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
-+{
-+}
-+
-+static void
-+fire_sched_out_preempt_notifiers(struct task_struct *curr,
-+ struct task_struct *next)
-+{
-+}
-+
-+#endif /* CONFIG_PREEMPT_NOTIFIERS */
-+
-+/**
-+ * prepare_task_switch - prepare to switch tasks
-+ * @rq: the runqueue preparing to switch
-+ * @prev: the current task that is being switched out
-+ * @next: the task we are going to switch to.
-+ *
-+ * This is called with the rq lock held and interrupts off. It must
-+ * be paired with a subsequent finish_task_switch after the context
-+ * switch.
-+ *
-+ * prepare_task_switch sets up locking and calls architecture specific
-+ * hooks.
-+ */
-+static inline void
-+prepare_task_switch(struct rq *rq, struct task_struct *prev,
-+ struct task_struct *next)
-+{
-+ trace_sched_switch(prev, next);
-+ sched_info_switch(rq, prev, next);
-+ perf_event_task_sched_out(prev, next);
-+ fire_sched_out_preempt_notifiers(prev, next);
-+ prepare_lock_switch(rq, next);
-+ prepare_arch_switch(next);
-+}
-+
-+/**
-+ * finish_task_switch - clean up after a task-switch
-+ * @prev: the thread we just switched away from.
-+ *
-+ * finish_task_switch must be called after the context switch, paired
-+ * with a prepare_task_switch call before the context switch.
-+ * finish_task_switch will reconcile locking set up by prepare_task_switch,
-+ * and do any other architecture-specific cleanup actions.
-+ *
-+ * Note that we may have delayed dropping an mm in context_switch(). If
-+ * so, we finish that here outside of the runqueue lock. (Doing it
-+ * with the lock held can cause deadlocks; see schedule() for
-+ * details.)
-+ *
-+ * The context switch have flipped the stack from under us and restored the
-+ * local variables which were saved when this task called schedule() in the
-+ * past. prev == current is still correct but we need to recalculate this_rq
-+ * because prev may have moved to another CPU.
-+ */
-+static struct rq *finish_task_switch(struct task_struct *prev)
-+ __releases(rq->lock)
-+{
-+ struct rq *rq = this_rq();
-+ struct mm_struct *mm = rq->prev_mm;
-+ long prev_state;
-+
-+ rq->prev_mm = NULL;
-+
-+ /*
-+ * A task struct has one reference for the use as "current".
-+ * If a task dies, then it sets TASK_DEAD in tsk->state and calls
-+ * schedule one last time. The schedule call will never return, and
-+ * the scheduled task must drop that reference.
-+ * The test for TASK_DEAD must occur while the runqueue locks are
-+ * still held, otherwise prev could be scheduled on another cpu, die
-+ * there before we look at prev->state, and then the reference would
-+ * be dropped twice.
-+ * Manfred Spraul <manfred@colorfullife.com>
-+ */
-+ prev_state = prev->state;
-+ vtime_task_switch(prev);
-+ finish_arch_switch(prev);
-+ perf_event_task_sched_in(prev, current);
-+ finish_lock_switch(rq, prev);
-+ finish_arch_post_lock_switch();
-+
-+ fire_sched_in_preempt_notifiers(current);
-+ if (mm)
-+ mmdrop(mm);
-+ if (unlikely(prev_state == TASK_DEAD)) {
-+ if (prev->sched_class->task_dead)
-+ prev->sched_class->task_dead(prev);
-+
-+ /*
-+ * Remove function-return probe instances associated with this
-+ * task and put them back on the free list.
-+ */
-+ kprobe_flush_task(prev);
-+ put_task_struct(prev);
-+ }
-+
-+ tick_nohz_task_switch(current);
-+ return rq;
-+}
-+
-+#ifdef CONFIG_SMP
-+
-+/* rq->lock is NOT held, but preemption is disabled */
-+static inline void post_schedule(struct rq *rq)
-+{
-+ if (rq->post_schedule) {
-+ unsigned long flags;
-+
-+ raw_spin_lock_irqsave(&rq->lock, flags);
-+ if (rq->curr->sched_class->post_schedule)
-+ rq->curr->sched_class->post_schedule(rq);
-+ raw_spin_unlock_irqrestore(&rq->lock, flags);
-+
-+ rq->post_schedule = 0;
-+ }
-+}
-+
-+#else
-+
-+static inline void post_schedule(struct rq *rq)
-+{
-+}
-+
-+#endif
-+
-+/**
-+ * schedule_tail - first thing a freshly forked thread must call.
-+ * @prev: the thread we just switched away from.
-+ */
-+asmlinkage __visible void schedule_tail(struct task_struct *prev)
-+ __releases(rq->lock)
-+{
-+ struct rq *rq;
-+
-+ /* finish_task_switch() drops rq->lock and enables preemtion */
-+ preempt_disable();
-+ rq = finish_task_switch(prev);
-+ post_schedule(rq);
-+ preempt_enable();
-+
-+ if (current->set_child_tid)
-+ put_user(task_pid_vnr(current), current->set_child_tid);
-+}
-+
-+/*
-+ * context_switch - switch to the new MM and the new thread's register state.
-+ */
-+static inline struct rq *
-+context_switch(struct rq *rq, struct task_struct *prev,
-+ struct task_struct *next)
-+{
-+ struct mm_struct *mm, *oldmm;
-+
-+ prepare_task_switch(rq, prev, next);
-+
-+ mm = next->mm;
-+ oldmm = prev->active_mm;
-+ /*
-+ * For paravirt, this is coupled with an exit in switch_to to
-+ * combine the page table reload and the switch backend into
-+ * one hypercall.
-+ */
-+ arch_start_context_switch(prev);
-+
-+ if (!mm) {
-+ next->active_mm = oldmm;
-+ atomic_inc(&oldmm->mm_count);
-+ enter_lazy_tlb(oldmm, next);
-+ } else
-+ switch_mm(oldmm, mm, next);
-+
-+ if (!prev->mm) {
-+ prev->active_mm = NULL;
-+ rq->prev_mm = oldmm;
-+ }
-+ /*
-+ * Since the runqueue lock will be released by the next
-+ * task (which is an invalid locking op but in the case
-+ * of the scheduler it's an obvious special-case), so we
-+ * do an early lockdep release here:
-+ */
-+ spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
-+
-+ context_tracking_task_switch(prev, next);
-+ /* Here we just switch the register state and the stack. */
-+ switch_to(prev, next, prev);
-+ barrier();
-+
-+ return finish_task_switch(prev);
-+}
-+
-+/*
-+ * nr_running and nr_context_switches:
-+ *
-+ * externally visible scheduler statistics: current number of runnable
-+ * threads, total number of context switches performed since bootup.
-+ */
-+unsigned long nr_running(void)
-+{
-+ unsigned long i, sum = 0;
-+
-+ for_each_online_cpu(i)
-+ sum += cpu_rq(i)->nr_running;
-+
-+ return sum;
-+}
-+
-+/*
-+ * Check if only the current task is running on the cpu.
-+ */
-+bool single_task_running(void)
-+{
-+ if (cpu_rq(smp_processor_id())->nr_running == 1)
-+ return true;
-+ else
-+ return false;
-+}
-+EXPORT_SYMBOL(single_task_running);
-+
-+unsigned long long nr_context_switches(void)
-+{
-+ int i;
-+ unsigned long long sum = 0;
-+
-+ for_each_possible_cpu(i)
-+ sum += cpu_rq(i)->nr_switches;
-+
-+ return sum;
-+}
-+
-+unsigned long nr_iowait(void)
-+{
-+ unsigned long i, sum = 0;
-+
-+ for_each_possible_cpu(i)
-+ sum += atomic_read(&cpu_rq(i)->nr_iowait);
-+
-+ return sum;
-+}
-+
-+unsigned long nr_iowait_cpu(int cpu)
-+{
-+ struct rq *this = cpu_rq(cpu);
-+ return atomic_read(&this->nr_iowait);
-+}
-+
-+void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
-+{
-+ struct rq *this = this_rq();
-+ *nr_waiters = atomic_read(&this->nr_iowait);
-+ *load = this->cpu_load[0];
-+}
-+
-+#ifdef CONFIG_SMP
-+
-+/*
-+ * sched_exec - execve() is a valuable balancing opportunity, because at
-+ * this point the task has the smallest effective memory and cache footprint.
-+ */
-+void sched_exec(void)
-+{
-+ struct task_struct *p = current;
-+ unsigned long flags;
-+ int dest_cpu;
-+
-+ raw_spin_lock_irqsave(&p->pi_lock, flags);
-+ dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
-+ if (dest_cpu == smp_processor_id())
-+ goto unlock;
-+
-+ if (likely(cpu_active(dest_cpu))) {
-+ struct migration_arg arg = { p, dest_cpu };
-+
-+ raw_spin_unlock_irqrestore(&p->pi_lock, flags);
-+ stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
-+ return;
-+ }
-+unlock:
-+ raw_spin_unlock_irqrestore(&p->pi_lock, flags);
-+}
-+
-+#endif
-+
-+DEFINE_PER_CPU(struct kernel_stat, kstat);
-+DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
-+
-+EXPORT_PER_CPU_SYMBOL(kstat);
-+EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
-+
-+/*
-+ * Return accounted runtime for the task.
-+ * In case the task is currently running, return the runtime plus current's
-+ * pending runtime that have not been accounted yet.
-+ */
-+unsigned long long task_sched_runtime(struct task_struct *p)
-+{
-+ unsigned long flags;
-+ struct rq *rq;
-+ u64 ns;
-+
-+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
-+ /*
-+ * 64-bit doesn't need locks to atomically read a 64bit value.
-+ * So we have a optimization chance when the task's delta_exec is 0.
-+ * Reading ->on_cpu is racy, but this is ok.
-+ *
-+ * If we race with it leaving cpu, we'll take a lock. So we're correct.
-+ * If we race with it entering cpu, unaccounted time is 0. This is
-+ * indistinguishable from the read occurring a few cycles earlier.
-+ * If we see ->on_cpu without ->on_rq, the task is leaving, and has
-+ * been accounted, so we're correct here as well.
-+ */
-+ if (!p->on_cpu || !task_on_rq_queued(p))
-+ return p->se.sum_exec_runtime;
-+#endif
-+
-+ rq = task_rq_lock(p, &flags);
-+ /*
-+ * Must be ->curr _and_ ->on_rq. If dequeued, we would
-+ * project cycles that may never be accounted to this
-+ * thread, breaking clock_gettime().
-+ */
-+ if (task_current(rq, p) && task_on_rq_queued(p)) {
-+ update_rq_clock(rq);
-+ p->sched_class->update_curr(rq);
-+ }
-+ ns = p->se.sum_exec_runtime;
-+ task_rq_unlock(rq, p, &flags);
-+
-+ return ns;
-+}
-+
-+/*
-+ * This function gets called by the timer code, with HZ frequency.
-+ * We call it with interrupts disabled.
-+ */
-+void scheduler_tick(void)
-+{
-+ int cpu = smp_processor_id();
-+ struct rq *rq = cpu_rq(cpu);
-+ struct task_struct *curr = rq->curr;
-+
-+ sched_clock_tick();
-+
-+ raw_spin_lock(&rq->lock);
-+ update_rq_clock(rq);
-+ curr->sched_class->task_tick(rq, curr, 0);
-+ update_cpu_load_active(rq);
-+ raw_spin_unlock(&rq->lock);
-+
-+ perf_event_task_tick();
-+
-+#ifdef CONFIG_SMP
-+ rq->idle_balance = idle_cpu(cpu);
-+ trigger_load_balance(rq);
-+#endif
-+ rq_last_tick_reset(rq);
-+}
-+
-+#ifdef CONFIG_NO_HZ_FULL
-+/**
-+ * scheduler_tick_max_deferment
-+ *
-+ * Keep at least one tick per second when a single
-+ * active task is running because the scheduler doesn't
-+ * yet completely support full dynticks environment.
-+ *
-+ * This makes sure that uptime, CFS vruntime, load
-+ * balancing, etc... continue to move forward, even
-+ * with a very low granularity.
-+ *
-+ * Return: Maximum deferment in nanoseconds.
-+ */
-+u64 scheduler_tick_max_deferment(void)
-+{
-+ struct rq *rq = this_rq();
-+ unsigned long next, now = ACCESS_ONCE(jiffies);
-+
-+ next = rq->last_sched_tick + HZ;
-+
-+ if (time_before_eq(next, now))
-+ return 0;
-+
-+ return jiffies_to_nsecs(next - now);
-+}
-+#endif
-+
-+notrace unsigned long get_parent_ip(unsigned long addr)
-+{
-+ if (in_lock_functions(addr)) {
-+ addr = CALLER_ADDR2;
-+ if (in_lock_functions(addr))
-+ addr = CALLER_ADDR3;
-+ }
-+ return addr;
-+}
-+
-+#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
-+ defined(CONFIG_PREEMPT_TRACER))
-+
-+void preempt_count_add(int val)
-+{
-+#ifdef CONFIG_DEBUG_PREEMPT
-+ /*
-+ * Underflow?
-+ */
-+ if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
-+ return;
-+#endif
-+ __preempt_count_add(val);
-+#ifdef CONFIG_DEBUG_PREEMPT
-+ /*
-+ * Spinlock count overflowing soon?
-+ */
-+ DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
-+ PREEMPT_MASK - 10);
-+#endif
-+ if (preempt_count() == val) {
-+ unsigned long ip = get_parent_ip(CALLER_ADDR1);
-+#ifdef CONFIG_DEBUG_PREEMPT
-+ current->preempt_disable_ip = ip;
-+#endif
-+ trace_preempt_off(CALLER_ADDR0, ip);
-+ }
-+}
-+EXPORT_SYMBOL(preempt_count_add);
-+NOKPROBE_SYMBOL(preempt_count_add);
-+
-+void preempt_count_sub(int val)
-+{
-+#ifdef CONFIG_DEBUG_PREEMPT
-+ /*
-+ * Underflow?
-+ */
-+ if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
-+ return;
-+ /*
-+ * Is the spinlock portion underflowing?
-+ */
-+ if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
-+ !(preempt_count() & PREEMPT_MASK)))
-+ return;
-+#endif
-+
-+ if (preempt_count() == val)
-+ trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
-+ __preempt_count_sub(val);
-+}
-+EXPORT_SYMBOL(preempt_count_sub);
-+NOKPROBE_SYMBOL(preempt_count_sub);
-+
-+#endif
-+
-+/*
-+ * Print scheduling while atomic bug:
-+ */
-+static noinline void __schedule_bug(struct task_struct *prev)
-+{
-+ if (oops_in_progress)
-+ return;
-+
-+ printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
-+ prev->comm, prev->pid, preempt_count());
-+
-+ debug_show_held_locks(prev);
-+ print_modules();
-+ if (irqs_disabled())
-+ print_irqtrace_events(prev);
-+#ifdef CONFIG_DEBUG_PREEMPT
-+ if (in_atomic_preempt_off()) {
-+ pr_err("Preemption disabled at:");
-+ print_ip_sym(current->preempt_disable_ip);
-+ pr_cont("\n");
-+ }
-+#endif
-+ dump_stack();
-+ add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
-+}
-+
-+/*
-+ * Various schedule()-time debugging checks and statistics:
-+ */
-+static inline void schedule_debug(struct task_struct *prev)
-+{
-+#ifdef CONFIG_SCHED_STACK_END_CHECK
-+ BUG_ON(unlikely(task_stack_end_corrupted(prev)));
-+#endif
-+ /*
-+ * Test if we are atomic. Since do_exit() needs to call into
-+ * schedule() atomically, we ignore that path. Otherwise whine
-+ * if we are scheduling when we should not.
-+ */
-+ if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD))
-+ __schedule_bug(prev);
-+ rcu_sleep_check();
-+
-+ profile_hit(SCHED_PROFILING, __builtin_return_address(0));
-+
-+ schedstat_inc(this_rq(), sched_count);
-+}
-+
-+/*
-+ * Pick up the highest-prio task:
-+ */
-+static inline struct task_struct *
-+pick_next_task(struct rq *rq, struct task_struct *prev)
-+{
-+ const struct sched_class *class = &fair_sched_class;
-+ struct task_struct *p;
-+
-+ /*
-+ * Optimization: we know that if all tasks are in
-+ * the fair class we can call that function directly:
-+ */
-+ if (likely(prev->sched_class == class &&
-+ rq->nr_running == rq->cfs.h_nr_running)) {
-+ p = fair_sched_class.pick_next_task(rq, prev);
-+ if (unlikely(p == RETRY_TASK))
-+ goto again;
-+
-+ /* assumes fair_sched_class->next == idle_sched_class */
-+ if (unlikely(!p))
-+ p = idle_sched_class.pick_next_task(rq, prev);
-+
-+ return p;
-+ }
-+
-+again:
-+ for_each_class(class) {
-+ p = class->pick_next_task(rq, prev);
-+ if (p) {
-+ if (unlikely(p == RETRY_TASK))
-+ goto again;
-+ return p;
-+ }
-+ }
-+
-+ BUG(); /* the idle class will always have a runnable task */
-+}
-+
-+/*
-+ * __schedule() is the main scheduler function.
-+ *
-+ * The main means of driving the scheduler and thus entering this function are:
-+ *
-+ * 1. Explicit blocking: mutex, semaphore, waitqueue, etc.
-+ *
-+ * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
-+ * paths. For example, see arch/x86/entry_64.S.
-+ *
-+ * To drive preemption between tasks, the scheduler sets the flag in timer
-+ * interrupt handler scheduler_tick().
-+ *
-+ * 3. Wakeups don't really cause entry into schedule(). They add a
-+ * task to the run-queue and that's it.
-+ *
-+ * Now, if the new task added to the run-queue preempts the current
-+ * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
-+ * called on the nearest possible occasion:
-+ *
-+ * - If the kernel is preemptible (CONFIG_PREEMPT=y):
-+ *
-+ * - in syscall or exception context, at the next outmost
-+ * preempt_enable(). (this might be as soon as the wake_up()'s
-+ * spin_unlock()!)
-+ *
-+ * - in IRQ context, return from interrupt-handler to
-+ * preemptible context
-+ *
-+ * - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
-+ * then at the next:
-+ *
-+ * - cond_resched() call
-+ * - explicit schedule() call
-+ * - return from syscall or exception to user-space
-+ * - return from interrupt-handler to user-space
-+ *
-+ * WARNING: all callers must re-check need_resched() afterward and reschedule
-+ * accordingly in case an event triggered the need for rescheduling (such as
-+ * an interrupt waking up a task) while preemption was disabled in __schedule().
-+ */
-+static void __sched __schedule(void)
-+{
-+ struct task_struct *prev, *next;
-+ unsigned long *switch_count;
-+ struct rq *rq;
-+ int cpu;
-+
-+ preempt_disable();
-+ cpu = smp_processor_id();
-+ rq = cpu_rq(cpu);
-+ rcu_note_context_switch();
-+ prev = rq->curr;
-+
-+ schedule_debug(prev);
-+
-+ if (sched_feat(HRTICK))
-+ hrtick_clear(rq);
-+
-+ /*
-+ * Make sure that signal_pending_state()->signal_pending() below
-+ * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
-+ * done by the caller to avoid the race with signal_wake_up().
-+ */
-+ smp_mb__before_spinlock();
-+ raw_spin_lock_irq(&rq->lock);
-+
-+ rq->clock_skip_update <<= 1; /* promote REQ to ACT */
-+
-+ switch_count = &prev->nivcsw;
-+ if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
-+ if (unlikely(signal_pending_state(prev->state, prev))) {
-+ prev->state = TASK_RUNNING;
-+ } else {
-+ deactivate_task(rq, prev, DEQUEUE_SLEEP);
-+ prev->on_rq = 0;
-+
-+ /*
-+ * If a worker went to sleep, notify and ask workqueue
-+ * whether it wants to wake up a task to maintain
-+ * concurrency.
-+ */
-+ if (prev->flags & PF_WQ_WORKER) {
-+ struct task_struct *to_wakeup;
-+
-+ to_wakeup = wq_worker_sleeping(prev, cpu);
-+ if (to_wakeup)
-+ try_to_wake_up_local(to_wakeup);
-+ }
-+ }
-+ switch_count = &prev->nvcsw;
-+ }
-+
-+ if (task_on_rq_queued(prev))
-+ update_rq_clock(rq);
-+
-+ next = pick_next_task(rq, prev);
-+ clear_tsk_need_resched(prev);
-+ clear_preempt_need_resched();
-+ rq->clock_skip_update = 0;
-+
-+ if (likely(prev != next)) {
-+ rq->nr_switches++;
-+ rq->curr = next;
-+ ++*switch_count;
-+
-+ rq = context_switch(rq, prev, next); /* unlocks the rq */
-+ cpu = cpu_of(rq);
-+ } else
-+ raw_spin_unlock_irq(&rq->lock);
-+
-+ post_schedule(rq);
-+
-+ sched_preempt_enable_no_resched();
-+}
-+
-+static inline void sched_submit_work(struct task_struct *tsk)
-+{
-+ if (!tsk->state || tsk_is_pi_blocked(tsk))
-+ return;
-+ /*
-+ * If we are going to sleep and we have plugged IO queued,
-+ * make sure to submit it to avoid deadlocks.
-+ */
-+ if (blk_needs_flush_plug(tsk))
-+ blk_schedule_flush_plug(tsk);
-+}
-+
-+asmlinkage __visible void __sched schedule(void)
-+{
-+ struct task_struct *tsk = current;
-+
-+ sched_submit_work(tsk);
-+ do {
-+ __schedule();
-+ } while (need_resched());
-+}
-+EXPORT_SYMBOL(schedule);
-+
-+#ifdef CONFIG_CONTEXT_TRACKING
-+asmlinkage __visible void __sched schedule_user(void)
-+{
-+ /*
-+ * If we come here after a random call to set_need_resched(),
-+ * or we have been woken up remotely but the IPI has not yet arrived,
-+ * we haven't yet exited the RCU idle mode. Do it here manually until
-+ * we find a better solution.
-+ *
-+ * NB: There are buggy callers of this function. Ideally we
-+ * should warn if prev_state != CONTEXT_USER, but that will trigger
-+ * too frequently to make sense yet.
-+ */
-+ enum ctx_state prev_state = exception_enter();
-+ schedule();
-+ exception_exit(prev_state);
-+}
-+#endif
-+
-+/**
-+ * schedule_preempt_disabled - called with preemption disabled
-+ *
-+ * Returns with preemption disabled. Note: preempt_count must be 1
-+ */
-+void __sched schedule_preempt_disabled(void)
-+{
-+ sched_preempt_enable_no_resched();
-+ schedule();
-+ preempt_disable();
-+}
-+
-+static void __sched notrace preempt_schedule_common(void)
-+{
-+ do {
-+ __preempt_count_add(PREEMPT_ACTIVE);
-+ __schedule();
-+ __preempt_count_sub(PREEMPT_ACTIVE);
-+
-+ /*
-+ * Check again in case we missed a preemption opportunity
-+ * between schedule and now.
-+ */
-+ barrier();
-+ } while (need_resched());
-+}
-+
-+#ifdef CONFIG_PREEMPT
-+/*
-+ * this is the entry point to schedule() from in-kernel preemption
-+ * off of preempt_enable. Kernel preemptions off return from interrupt
-+ * occur there and call schedule directly.
-+ */
-+asmlinkage __visible void __sched notrace preempt_schedule(void)
-+{
-+ /*
-+ * If there is a non-zero preempt_count or interrupts are disabled,
-+ * we do not want to preempt the current task. Just return..
-+ */
-+ if (likely(!preemptible()))
-+ return;
-+
-+ preempt_schedule_common();
-+}
-+NOKPROBE_SYMBOL(preempt_schedule);
-+EXPORT_SYMBOL(preempt_schedule);
-+
-+#ifdef CONFIG_CONTEXT_TRACKING
-+/**
-+ * preempt_schedule_context - preempt_schedule called by tracing
-+ *
-+ * The tracing infrastructure uses preempt_enable_notrace to prevent
-+ * recursion and tracing preempt enabling caused by the tracing
-+ * infrastructure itself. But as tracing can happen in areas coming
-+ * from userspace or just about to enter userspace, a preempt enable
-+ * can occur before user_exit() is called. This will cause the scheduler
-+ * to be called when the system is still in usermode.
-+ *
-+ * To prevent this, the preempt_enable_notrace will use this function
-+ * instead of preempt_schedule() to exit user context if needed before
-+ * calling the scheduler.
-+ */
-+asmlinkage __visible void __sched notrace preempt_schedule_context(void)
-+{
-+ enum ctx_state prev_ctx;
-+
-+ if (likely(!preemptible()))
-+ return;
-+
-+ do {
-+ __preempt_count_add(PREEMPT_ACTIVE);
-+ /*
-+ * Needs preempt disabled in case user_exit() is traced
-+ * and the tracer calls preempt_enable_notrace() causing
-+ * an infinite recursion.
-+ */
-+ prev_ctx = exception_enter();
-+ __schedule();
-+ exception_exit(prev_ctx);
-+
-+ __preempt_count_sub(PREEMPT_ACTIVE);
-+ barrier();
-+ } while (need_resched());
-+}
-+EXPORT_SYMBOL_GPL(preempt_schedule_context);
-+#endif /* CONFIG_CONTEXT_TRACKING */
-+
-+#endif /* CONFIG_PREEMPT */
-+
-+/*
-+ * this is the entry point to schedule() from kernel preemption
-+ * off of irq context.
-+ * Note, that this is called and return with irqs disabled. This will
-+ * protect us against recursive calling from irq.
-+ */
-+asmlinkage __visible void __sched preempt_schedule_irq(void)
-+{
-+ enum ctx_state prev_state;
-+
-+ /* Catch callers which need to be fixed */
-+ BUG_ON(preempt_count() || !irqs_disabled());
-+
-+ prev_state = exception_enter();
-+
-+ do {
-+ __preempt_count_add(PREEMPT_ACTIVE);
-+ local_irq_enable();
-+ __schedule();
-+ local_irq_disable();
-+ __preempt_count_sub(PREEMPT_ACTIVE);
-+
-+ /*
-+ * Check again in case we missed a preemption opportunity
-+ * between schedule and now.
-+ */
-+ barrier();
-+ } while (need_resched());
-+
-+ exception_exit(prev_state);
-+}
-+
-+int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
-+ void *key)
-+{
-+ return try_to_wake_up(curr->private, mode, wake_flags);
-+}
-+EXPORT_SYMBOL(default_wake_function);
-+
-+#ifdef CONFIG_RT_MUTEXES
-+
-+/*
-+ * rt_mutex_setprio - set the current priority of a task
-+ * @p: task
-+ * @prio: prio value (kernel-internal form)
-+ *
-+ * This function changes the 'effective' priority of a task. It does
-+ * not touch ->normal_prio like __setscheduler().
-+ *
-+ * Used by the rt_mutex code to implement priority inheritance
-+ * logic. Call site only calls if the priority of the task changed.
-+ */
-+void rt_mutex_setprio(struct task_struct *p, int prio)
-+{
-+ int oldprio, queued, running, enqueue_flag = 0;
-+ struct rq *rq;
-+ const struct sched_class *prev_class;
-+
-+ BUG_ON(prio > MAX_PRIO);
-+
-+ rq = __task_rq_lock(p);
-+
-+ /*
-+ * Idle task boosting is a nono in general. There is one
-+ * exception, when PREEMPT_RT and NOHZ is active:
-+ *
-+ * The idle task calls get_next_timer_interrupt() and holds
-+ * the timer wheel base->lock on the CPU and another CPU wants
-+ * to access the timer (probably to cancel it). We can safely
-+ * ignore the boosting request, as the idle CPU runs this code
-+ * with interrupts disabled and will complete the lock
-+ * protected section without being interrupted. So there is no
-+ * real need to boost.
-+ */
-+ if (unlikely(p == rq->idle)) {
-+ WARN_ON(p != rq->curr);
-+ WARN_ON(p->pi_blocked_on);
-+ goto out_unlock;
-+ }
-+
-+ trace_sched_pi_setprio(p, prio);
-+ oldprio = p->prio;
-+ prev_class = p->sched_class;
-+ queued = task_on_rq_queued(p);
-+ running = task_current(rq, p);
-+ if (queued)
-+ dequeue_task(rq, p, 0);
-+ if (running)
-+ put_prev_task(rq, p);
-+
-+ /*
-+ * Boosting condition are:
-+ * 1. -rt task is running and holds mutex A
-+ * --> -dl task blocks on mutex A
-+ *
-+ * 2. -dl task is running and holds mutex A
-+ * --> -dl task blocks on mutex A and could preempt the
-+ * running task
-+ */
-+ if (dl_prio(prio)) {
-+ struct task_struct *pi_task = rt_mutex_get_top_task(p);
-+ if (!dl_prio(p->normal_prio) ||
-+ (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
-+ p->dl.dl_boosted = 1;
-+ p->dl.dl_throttled = 0;
-+ enqueue_flag = ENQUEUE_REPLENISH;
-+ } else
-+ p->dl.dl_boosted = 0;
-+ p->sched_class = &dl_sched_class;
-+ } else if (rt_prio(prio)) {
-+ if (dl_prio(oldprio))
-+ p->dl.dl_boosted = 0;
-+ if (oldprio < prio)
-+ enqueue_flag = ENQUEUE_HEAD;
-+ p->sched_class = &rt_sched_class;
-+ } else {
-+ if (dl_prio(oldprio))
-+ p->dl.dl_boosted = 0;
-+ if (rt_prio(oldprio))
-+ p->rt.timeout = 0;
-+ p->sched_class = &fair_sched_class;
-+ }
-+
-+ p->prio = prio;
-+
-+ if (running)
-+ p->sched_class->set_curr_task(rq);
-+ if (queued)
-+ enqueue_task(rq, p, enqueue_flag);
-+
-+ check_class_changed(rq, p, prev_class, oldprio);
-+out_unlock:
-+ __task_rq_unlock(rq);
-+}
-+#endif
-+
-+void set_user_nice(struct task_struct *p, long nice)
-+{
-+ int old_prio, delta, queued;
-+ unsigned long flags;
-+ struct rq *rq;
-+
-+ if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
-+ return;
-+ /*
-+ * We have to be careful, if called from sys_setpriority(),
-+ * the task might be in the middle of scheduling on another CPU.
-+ */
-+ rq = task_rq_lock(p, &flags);
-+ /*
-+ * The RT priorities are set via sched_setscheduler(), but we still
-+ * allow the 'normal' nice value to be set - but as expected
-+ * it wont have any effect on scheduling until the task is
-+ * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
-+ */
-+ if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
-+ p->static_prio = NICE_TO_PRIO(nice);
-+ goto out_unlock;
-+ }
-+ queued = task_on_rq_queued(p);
-+ if (queued)
-+ dequeue_task(rq, p, 0);
-+
-+ p->static_prio = NICE_TO_PRIO(nice);
-+ set_load_weight(p);
-+ old_prio = p->prio;
-+ p->prio = effective_prio(p);
-+ delta = p->prio - old_prio;
-+
-+ if (queued) {
-+ enqueue_task(rq, p, 0);
-+ /*
-+ * If the task increased its priority or is running and
-+ * lowered its priority, then reschedule its CPU:
-+ */
-+ if (delta < 0 || (delta > 0 && task_running(rq, p)))
-+ resched_curr(rq);
-+ }
-+out_unlock:
-+ task_rq_unlock(rq, p, &flags);
-+}
-+EXPORT_SYMBOL(set_user_nice);
-+
-+/*
-+ * can_nice - check if a task can reduce its nice value
-+ * @p: task
-+ * @nice: nice value
-+ */
-+int can_nice(const struct task_struct *p, const int nice)
-+{
-+ /* convert nice value [19,-20] to rlimit style value [1,40] */
-+ int nice_rlim = nice_to_rlimit(nice);
-+
-+ return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
-+ capable(CAP_SYS_NICE));
-+}
-+
-+#ifdef __ARCH_WANT_SYS_NICE
-+
-+/*
-+ * sys_nice - change the priority of the current process.
-+ * @increment: priority increment
-+ *
-+ * sys_setpriority is a more generic, but much slower function that
-+ * does similar things.
-+ */
-+SYSCALL_DEFINE1(nice, int, increment)
-+{
-+ long nice, retval;
-+
-+ /*
-+ * Setpriority might change our priority at the same moment.
-+ * We don't have to worry. Conceptually one call occurs first
-+ * and we have a single winner.
-+ */
-+ increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
-+ nice = task_nice(current) + increment;
-+
-+ nice = clamp_val(nice, MIN_NICE, MAX_NICE);
-+ if (increment < 0 && !can_nice(current, nice))
-+ return -EPERM;
-+
-+ retval = security_task_setnice(current, nice);
-+ if (retval)
-+ return retval;
-+
-+ set_user_nice(current, nice);
-+ return 0;
-+}
-+
-+#endif
-+
-+/**
-+ * task_prio - return the priority value of a given task.
-+ * @p: the task in question.
-+ *
-+ * Return: The priority value as seen by users in /proc.
-+ * RT tasks are offset by -200. Normal tasks are centered
-+ * around 0, value goes from -16 to +15.
-+ */
-+int task_prio(const struct task_struct *p)
-+{
-+ return p->prio - MAX_RT_PRIO;
-+}
-+
-+/**
-+ * idle_cpu - is a given cpu idle currently?
-+ * @cpu: the processor in question.
-+ *
-+ * Return: 1 if the CPU is currently idle. 0 otherwise.
-+ */
-+int idle_cpu(int cpu)
-+{
-+ struct rq *rq = cpu_rq(cpu);
-+
-+ if (rq->curr != rq->idle)
-+ return 0;
-+
-+ if (rq->nr_running)
-+ return 0;
-+
-+#ifdef CONFIG_SMP
-+ if (!llist_empty(&rq->wake_list))
-+ return 0;
-+#endif
-+
-+ return 1;
-+}
-+
-+/**
-+ * idle_task - return the idle task for a given cpu.
-+ * @cpu: the processor in question.
-+ *
-+ * Return: The idle task for the cpu @cpu.
-+ */
-+struct task_struct *idle_task(int cpu)
-+{
-+ return cpu_rq(cpu)->idle;
-+}
-+
-+/**
-+ * find_process_by_pid - find a process with a matching PID value.
-+ * @pid: the pid in question.
-+ *
-+ * The task of @pid, if found. %NULL otherwise.
-+ */
-+static struct task_struct *find_process_by_pid(pid_t pid)
-+{
-+ return pid ? find_task_by_vpid(pid) : current;
-+}
-+
-+/*
-+ * This function initializes the sched_dl_entity of a newly becoming
-+ * SCHED_DEADLINE task.
-+ *
-+ * Only the static values are considered here, the actual runtime and the
-+ * absolute deadline will be properly calculated when the task is enqueued
-+ * for the first time with its new policy.
-+ */
-+static void
-+__setparam_dl(struct task_struct *p, const struct sched_attr *attr)
-+{
-+ struct sched_dl_entity *dl_se = &p->dl;
-+
-+ dl_se->dl_runtime = attr->sched_runtime;
-+ dl_se->dl_deadline = attr->sched_deadline;
-+ dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
-+ dl_se->flags = attr->sched_flags;
-+ dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
-+
-+ /*
-+ * Changing the parameters of a task is 'tricky' and we're not doing
-+ * the correct thing -- also see task_dead_dl() and switched_from_dl().
-+ *
-+ * What we SHOULD do is delay the bandwidth release until the 0-lag
-+ * point. This would include retaining the task_struct until that time
-+ * and change dl_overflow() to not immediately decrement the current
-+ * amount.
-+ *
-+ * Instead we retain the current runtime/deadline and let the new
-+ * parameters take effect after the current reservation period lapses.
-+ * This is safe (albeit pessimistic) because the 0-lag point is always
-+ * before the current scheduling deadline.
-+ *
-+ * We can still have temporary overloads because we do not delay the
-+ * change in bandwidth until that time; so admission control is
-+ * not on the safe side. It does however guarantee tasks will never
-+ * consume more than promised.
-+ */
-+}
-+
-+/*
-+ * sched_setparam() passes in -1 for its policy, to let the functions
-+ * it calls know not to change it.
-+ */
-+#define SETPARAM_POLICY -1
-+
-+static void __setscheduler_params(struct task_struct *p,
-+ const struct sched_attr *attr)
-+{
-+ int policy = attr->sched_policy;
-+
-+ if (policy == SETPARAM_POLICY)
-+ policy = p->policy;
-+
-+ p->policy = policy;
-+
-+ if (dl_policy(policy))
-+ __setparam_dl(p, attr);
-+ else if (fair_policy(policy))
-+ p->static_prio = NICE_TO_PRIO(attr->sched_nice);
-+
-+ /*
-+ * __sched_setscheduler() ensures attr->sched_priority == 0 when
-+ * !rt_policy. Always setting this ensures that things like
-+ * getparam()/getattr() don't report silly values for !rt tasks.
-+ */
-+ p->rt_priority = attr->sched_priority;
-+ p->normal_prio = normal_prio(p);
-+ set_load_weight(p);
-+}
-+
-+/* Actually do priority change: must hold pi & rq lock. */
-+static void __setscheduler(struct rq *rq, struct task_struct *p,
-+ const struct sched_attr *attr, bool keep_boost)
-+{
-+ __setscheduler_params(p, attr);
-+
-+ /*
-+ * Keep a potential priority boosting if called from
-+ * sched_setscheduler().
-+ */
-+ if (keep_boost)
-+ p->prio = rt_mutex_get_effective_prio(p, normal_prio(p));
-+ else
-+ p->prio = normal_prio(p);
-+
-+ if (dl_prio(p->prio))
-+ p->sched_class = &dl_sched_class;
-+ else if (rt_prio(p->prio))
-+ p->sched_class = &rt_sched_class;
-+ else
-+ p->sched_class = &fair_sched_class;
-+}
-+
-+static void
-+__getparam_dl(struct task_struct *p, struct sched_attr *attr)
-+{
-+ struct sched_dl_entity *dl_se = &p->dl;
-+
-+ attr->sched_priority = p->rt_priority;
-+ attr->sched_runtime = dl_se->dl_runtime;
-+ attr->sched_deadline = dl_se->dl_deadline;
-+ attr->sched_period = dl_se->dl_period;
-+ attr->sched_flags = dl_se->flags;
-+}
-+
-+/*
-+ * This function validates the new parameters of a -deadline task.
-+ * We ask for the deadline not being zero, and greater or equal
-+ * than the runtime, as well as the period of being zero or
-+ * greater than deadline. Furthermore, we have to be sure that
-+ * user parameters are above the internal resolution of 1us (we
-+ * check sched_runtime only since it is always the smaller one) and
-+ * below 2^63 ns (we have to check both sched_deadline and
-+ * sched_period, as the latter can be zero).
-+ */
-+static bool
-+__checkparam_dl(const struct sched_attr *attr)
-+{
-+ /* deadline != 0 */
-+ if (attr->sched_deadline == 0)
-+ return false;
-+
-+ /*
-+ * Since we truncate DL_SCALE bits, make sure we're at least
-+ * that big.
-+ */
-+ if (attr->sched_runtime < (1ULL << DL_SCALE))
-+ return false;
-+
-+ /*
-+ * Since we use the MSB for wrap-around and sign issues, make
-+ * sure it's not set (mind that period can be equal to zero).
-+ */
-+ if (attr->sched_deadline & (1ULL << 63) ||
-+ attr->sched_period & (1ULL << 63))
-+ return false;
-+
-+ /* runtime <= deadline <= period (if period != 0) */
-+ if ((attr->sched_period != 0 &&
-+ attr->sched_period < attr->sched_deadline) ||
-+ attr->sched_deadline < attr->sched_runtime)
-+ return false;
-+
-+ return true;
-+}
-+
-+/*
-+ * check the target process has a UID that matches the current process's
-+ */
-+static bool check_same_owner(struct task_struct *p)
-+{
-+ const struct cred *cred = current_cred(), *pcred;
-+ bool match;
-+
-+ rcu_read_lock();
-+ pcred = __task_cred(p);
-+ match = (uid_eq(cred->euid, pcred->euid) ||
-+ uid_eq(cred->euid, pcred->uid));
-+ rcu_read_unlock();
-+ return match;
-+}
-+
-+static bool dl_param_changed(struct task_struct *p,
-+ const struct sched_attr *attr)
-+{
-+ struct sched_dl_entity *dl_se = &p->dl;
-+
-+ if (dl_se->dl_runtime != attr->sched_runtime ||
-+ dl_se->dl_deadline != attr->sched_deadline ||
-+ dl_se->dl_period != attr->sched_period ||
-+ dl_se->flags != attr->sched_flags)
-+ return true;
-+
-+ return false;
-+}
-+
-+static int __sched_setscheduler(struct task_struct *p,
-+ const struct sched_attr *attr,
-+ bool user)
-+{
-+ int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
-+ MAX_RT_PRIO - 1 - attr->sched_priority;
-+ int retval, oldprio, oldpolicy = -1, queued, running;
-+ int new_effective_prio, policy = attr->sched_policy;
-+ unsigned long flags;
-+ const struct sched_class *prev_class;
-+ struct rq *rq;
-+ int reset_on_fork;
-+
-+ /* may grab non-irq protected spin_locks */
-+ BUG_ON(in_interrupt());
-+recheck:
-+ /* double check policy once rq lock held */
-+ if (policy < 0) {
-+ reset_on_fork = p->sched_reset_on_fork;
-+ policy = oldpolicy = p->policy;
-+ } else {
-+ reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
-+
-+ if (policy != SCHED_DEADLINE &&
-+ policy != SCHED_FIFO && policy != SCHED_RR &&
-+ policy != SCHED_NORMAL && policy != SCHED_BATCH &&
-+ policy != SCHED_IDLE)
-+ return -EINVAL;
-+ }
-+
-+ if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK))
-+ return -EINVAL;
-+
-+ /*
-+ * Valid priorities for SCHED_FIFO and SCHED_RR are
-+ * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
-+ * SCHED_BATCH and SCHED_IDLE is 0.
-+ */
-+ if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
-+ (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
-+ return -EINVAL;
-+ if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
-+ (rt_policy(policy) != (attr->sched_priority != 0)))
-+ return -EINVAL;
-+
-+ /*
-+ * Allow unprivileged RT tasks to decrease priority:
-+ */
-+ if (user && !capable(CAP_SYS_NICE)) {
-+ if (fair_policy(policy)) {
-+ if (attr->sched_nice < task_nice(p) &&
-+ !can_nice(p, attr->sched_nice))
-+ return -EPERM;
-+ }
-+
-+ if (rt_policy(policy)) {
-+ unsigned long rlim_rtprio =
-+ task_rlimit(p, RLIMIT_RTPRIO);
-+
-+ /* can't set/change the rt policy */
-+ if (policy != p->policy && !rlim_rtprio)
-+ return -EPERM;
-+
-+ /* can't increase priority */
-+ if (attr->sched_priority > p->rt_priority &&
-+ attr->sched_priority > rlim_rtprio)
-+ return -EPERM;
-+ }
-+
-+ /*
-+ * Can't set/change SCHED_DEADLINE policy at all for now
-+ * (safest behavior); in the future we would like to allow
-+ * unprivileged DL tasks to increase their relative deadline
-+ * or reduce their runtime (both ways reducing utilization)
-+ */
-+ if (dl_policy(policy))
-+ return -EPERM;
-+
-+ /*
-+ * Treat SCHED_IDLE as nice 20. Only allow a switch to
-+ * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
-+ */
-+ if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
-+ if (!can_nice(p, task_nice(p)))
-+ return -EPERM;
-+ }
-+
-+ /* can't change other user's priorities */
-+ if (!check_same_owner(p))
-+ return -EPERM;
-+
-+ /* Normal users shall not reset the sched_reset_on_fork flag */
-+ if (p->sched_reset_on_fork && !reset_on_fork)
-+ return -EPERM;
-+ }
-+
-+ if (user) {
-+ retval = security_task_setscheduler(p);
-+ if (retval)
-+ return retval;
-+ }
-+
-+ /*
-+ * make sure no PI-waiters arrive (or leave) while we are
-+ * changing the priority of the task:
-+ *
-+ * To be able to change p->policy safely, the appropriate
-+ * runqueue lock must be held.
-+ */
-+ rq = task_rq_lock(p, &flags);
-+
-+ /*
-+ * Changing the policy of the stop threads its a very bad idea
-+ */
-+ if (p == rq->stop) {
-+ task_rq_unlock(rq, p, &flags);
-+ return -EINVAL;
-+ }
-+
-+ /*
-+ * If not changing anything there's no need to proceed further,
-+ * but store a possible modification of reset_on_fork.
-+ */
-+ if (unlikely(policy == p->policy)) {
-+ if (fair_policy(policy) && attr->sched_nice != task_nice(p))
-+ goto change;
-+ if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
-+ goto change;
-+ if (dl_policy(policy) && dl_param_changed(p, attr))
-+ goto change;
-+
-+ p->sched_reset_on_fork = reset_on_fork;
-+ task_rq_unlock(rq, p, &flags);
-+ return 0;
-+ }
-+change:
-+
-+ if (user) {
-+#ifdef CONFIG_RT_GROUP_SCHED
-+ /*
-+ * Do not allow realtime tasks into groups that have no runtime
-+ * assigned.
-+ */
-+ if (rt_bandwidth_enabled() && rt_policy(policy) &&
-+ task_group(p)->rt_bandwidth.rt_runtime == 0 &&
-+ !task_group_is_autogroup(task_group(p))) {
-+ task_rq_unlock(rq, p, &flags);
-+ return -EPERM;
-+ }
-+#endif
-+#ifdef CONFIG_SMP
-+ if (dl_bandwidth_enabled() && dl_policy(policy)) {
-+ cpumask_t *span = rq->rd->span;
-+
-+ /*
-+ * Don't allow tasks with an affinity mask smaller than
-+ * the entire root_domain to become SCHED_DEADLINE. We
-+ * will also fail if there's no bandwidth available.
-+ */
-+ if (!cpumask_subset(span, &p->cpus_allowed) ||
-+ rq->rd->dl_bw.bw == 0) {
-+ task_rq_unlock(rq, p, &flags);
-+ return -EPERM;
-+ }
-+ }
-+#endif
-+ }
-+
-+ /* recheck policy now with rq lock held */
-+ if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
-+ policy = oldpolicy = -1;
-+ task_rq_unlock(rq, p, &flags);
-+ goto recheck;
-+ }
-+
-+ /*
-+ * If setscheduling to SCHED_DEADLINE (or changing the parameters
-+ * of a SCHED_DEADLINE task) we need to check if enough bandwidth
-+ * is available.
-+ */
-+ if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {
-+ task_rq_unlock(rq, p, &flags);
-+ return -EBUSY;
-+ }
-+
-+ p->sched_reset_on_fork = reset_on_fork;
-+ oldprio = p->prio;
-+
-+ /*
-+ * Take priority boosted tasks into account. If the new
-+ * effective priority is unchanged, we just store the new
-+ * normal parameters and do not touch the scheduler class and
-+ * the runqueue. This will be done when the task deboost
-+ * itself.
-+ */
-+ new_effective_prio = rt_mutex_get_effective_prio(p, newprio);
-+ if (new_effective_prio == oldprio) {
-+ __setscheduler_params(p, attr);
-+ task_rq_unlock(rq, p, &flags);
-+ return 0;
-+ }
-+
-+ queued = task_on_rq_queued(p);
-+ running = task_current(rq, p);
-+ if (queued)
-+ dequeue_task(rq, p, 0);
-+ if (running)
-+ put_prev_task(rq, p);
-+
-+ prev_class = p->sched_class;
-+ __setscheduler(rq, p, attr, true);
-+
-+ if (running)
-+ p->sched_class->set_curr_task(rq);
-+ if (queued) {
-+ /*
-+ * We enqueue to tail when the priority of a task is
-+ * increased (user space view).
-+ */
-+ enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0);
-+ }
-+
-+ check_class_changed(rq, p, prev_class, oldprio);
-+ task_rq_unlock(rq, p, &flags);
-+
-+ rt_mutex_adjust_pi(p);
-+
-+ return 0;
-+}
-+
-+static int _sched_setscheduler(struct task_struct *p, int policy,
-+ const struct sched_param *param, bool check)
-+{
-+ struct sched_attr attr = {
-+ .sched_policy = policy,
-+ .sched_priority = param->sched_priority,
-+ .sched_nice = PRIO_TO_NICE(p->static_prio),
-+ };
-+
-+ /* Fixup the legacy SCHED_RESET_ON_FORK hack. */
-+ if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
-+ attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
-+ policy &= ~SCHED_RESET_ON_FORK;
-+ attr.sched_policy = policy;
-+ }
-+
-+ return __sched_setscheduler(p, &attr, check);
-+}
-+/**
-+ * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
-+ * @p: the task in question.
-+ * @policy: new policy.
-+ * @param: structure containing the new RT priority.
-+ *
-+ * Return: 0 on success. An error code otherwise.
-+ *
-+ * NOTE that the task may be already dead.
-+ */
-+int sched_setscheduler(struct task_struct *p, int policy,
-+ const struct sched_param *param)
-+{
-+ return _sched_setscheduler(p, policy, param, true);
-+}
-+EXPORT_SYMBOL_GPL(sched_setscheduler);
-+
-+int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
-+{
-+ return __sched_setscheduler(p, attr, true);
-+}
-+EXPORT_SYMBOL_GPL(sched_setattr);
-+
-+/**
-+ * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
-+ * @p: the task in question.
-+ * @policy: new policy.
-+ * @param: structure containing the new RT priority.
-+ *
-+ * Just like sched_setscheduler, only don't bother checking if the
-+ * current context has permission. For example, this is needed in
-+ * stop_machine(): we create temporary high priority worker threads,
-+ * but our caller might not have that capability.
-+ *
-+ * Return: 0 on success. An error code otherwise.
-+ */
-+int sched_setscheduler_nocheck(struct task_struct *p, int policy,
-+ const struct sched_param *param)
-+{
-+ return _sched_setscheduler(p, policy, param, false);
-+}
-+
-+static int
-+do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
-+{
-+ struct sched_param lparam;
-+ struct task_struct *p;
-+ int retval;
-+
-+ if (!param || pid < 0)
-+ return -EINVAL;
-+ if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
-+ return -EFAULT;
-+
-+ rcu_read_lock();
-+ retval = -ESRCH;
-+ p = find_process_by_pid(pid);
-+ if (p != NULL)
-+ retval = sched_setscheduler(p, policy, &lparam);
-+ rcu_read_unlock();
-+
-+ return retval;
-+}
-+
-+/*
-+ * Mimics kernel/events/core.c perf_copy_attr().
-+ */
-+static int sched_copy_attr(struct sched_attr __user *uattr,
-+ struct sched_attr *attr)
-+{
-+ u32 size;
-+ int ret;
-+
-+ if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
-+ return -EFAULT;
-+
-+ /*
-+ * zero the full structure, so that a short copy will be nice.
-+ */
-+ memset(attr, 0, sizeof(*attr));
-+
-+ ret = get_user(size, &uattr->size);
-+ if (ret)
-+ return ret;
-+
-+ if (size > PAGE_SIZE) /* silly large */
-+ goto err_size;
-+
-+ if (!size) /* abi compat */
-+ size = SCHED_ATTR_SIZE_VER0;
-+
-+ if (size < SCHED_ATTR_SIZE_VER0)
-+ goto err_size;
-+
-+ /*
-+ * If we're handed a bigger struct than we know of,
-+ * ensure all the unknown bits are 0 - i.e. new
-+ * user-space does not rely on any kernel feature
-+ * extensions we dont know about yet.
-+ */
-+ if (size > sizeof(*attr)) {
-+ unsigned char __user *addr;
-+ unsigned char __user *end;
-+ unsigned char val;
-+
-+ addr = (void __user *)uattr + sizeof(*attr);
-+ end = (void __user *)uattr + size;
-+
-+ for (; addr < end; addr++) {
-+ ret = get_user(val, addr);
-+ if (ret)
-+ return ret;
-+ if (val)
-+ goto err_size;
-+ }
-+ size = sizeof(*attr);
-+ }
-+
-+ ret = copy_from_user(attr, uattr, size);
-+ if (ret)
-+ return -EFAULT;
-+
-+ /*
-+ * XXX: do we want to be lenient like existing syscalls; or do we want
-+ * to be strict and return an error on out-of-bounds values?
-+ */
-+ attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
-+
-+ return 0;
-+
-+err_size:
-+ put_user(sizeof(*attr), &uattr->size);
-+ return -E2BIG;
-+}
-+
-+/**
-+ * sys_sched_setscheduler - set/change the scheduler policy and RT priority
-+ * @pid: the pid in question.
-+ * @policy: new policy.
-+ * @param: structure containing the new RT priority.
-+ *
-+ * Return: 0 on success. An error code otherwise.
-+ */
-+SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
-+ struct sched_param __user *, param)
-+{
-+ /* negative values for policy are not valid */
-+ if (policy < 0)
-+ return -EINVAL;
-+
-+ return do_sched_setscheduler(pid, policy, param);
-+}
-+
-+/**
-+ * sys_sched_setparam - set/change the RT priority of a thread
-+ * @pid: the pid in question.
-+ * @param: structure containing the new RT priority.
-+ *
-+ * Return: 0 on success. An error code otherwise.
-+ */
-+SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
-+{
-+ return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
-+}
-+
-+/**
-+ * sys_sched_setattr - same as above, but with extended sched_attr
-+ * @pid: the pid in question.
-+ * @uattr: structure containing the extended parameters.
-+ * @flags: for future extension.
-+ */
-+SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
-+ unsigned int, flags)
-+{
-+ struct sched_attr attr;
-+ struct task_struct *p;
-+ int retval;
-+
-+ if (!uattr || pid < 0 || flags)
-+ return -EINVAL;
-+
-+ retval = sched_copy_attr(uattr, &attr);
-+ if (retval)
-+ return retval;
-+
-+ if ((int)attr.sched_policy < 0)
-+ return -EINVAL;
-+
-+ rcu_read_lock();
-+ retval = -ESRCH;
-+ p = find_process_by_pid(pid);
-+ if (p != NULL)
-+ retval = sched_setattr(p, &attr);
-+ rcu_read_unlock();
-+
-+ return retval;
-+}
-+
-+/**
-+ * sys_sched_getscheduler - get the policy (scheduling class) of a thread
-+ * @pid: the pid in question.
-+ *
-+ * Return: On success, the policy of the thread. Otherwise, a negative error
-+ * code.
-+ */
-+SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
-+{
-+ struct task_struct *p;
-+ int retval;
-+
-+ if (pid < 0)
-+ return -EINVAL;
-+
-+ retval = -ESRCH;
-+ rcu_read_lock();
-+ p = find_process_by_pid(pid);
-+ if (p) {
-+ retval = security_task_getscheduler(p);
-+ if (!retval)
-+ retval = p->policy
-+ | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
-+ }
-+ rcu_read_unlock();
-+ return retval;
-+}
-+
-+/**
-+ * sys_sched_getparam - get the RT priority of a thread
-+ * @pid: the pid in question.
-+ * @param: structure containing the RT priority.
-+ *
-+ * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
-+ * code.
-+ */
-+SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
-+{
-+ struct sched_param lp = { .sched_priority = 0 };
-+ struct task_struct *p;
-+ int retval;
-+
-+ if (!param || pid < 0)
-+ return -EINVAL;
-+
-+ rcu_read_lock();
-+ p = find_process_by_pid(pid);
-+ retval = -ESRCH;
-+ if (!p)
-+ goto out_unlock;
-+
-+ retval = security_task_getscheduler(p);
-+ if (retval)
-+ goto out_unlock;
-+
-+ if (task_has_rt_policy(p))
-+ lp.sched_priority = p->rt_priority;
-+ rcu_read_unlock();
-+
-+ /*
-+ * This one might sleep, we cannot do it with a spinlock held ...
-+ */
-+ retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
-+
-+ return retval;
-+
-+out_unlock:
-+ rcu_read_unlock();
-+ return retval;
-+}
-+
-+static int sched_read_attr(struct sched_attr __user *uattr,
-+ struct sched_attr *attr,
-+ unsigned int usize)
-+{
-+ int ret;
-+
-+ if (!access_ok(VERIFY_WRITE, uattr, usize))
-+ return -EFAULT;
-+
-+ /*
-+ * If we're handed a smaller struct than we know of,
-+ * ensure all the unknown bits are 0 - i.e. old
-+ * user-space does not get uncomplete information.
-+ */
-+ if (usize < sizeof(*attr)) {
-+ unsigned char *addr;
-+ unsigned char *end;
-+
-+ addr = (void *)attr + usize;
-+ end = (void *)attr + sizeof(*attr);
-+
-+ for (; addr < end; addr++) {
-+ if (*addr)
-+ return -EFBIG;
-+ }
-+
-+ attr->size = usize;
-+ }
-+
-+ ret = copy_to_user(uattr, attr, attr->size);
-+ if (ret)
-+ return -EFAULT;
-+
-+ return 0;
-+}
-+
-+/**
-+ * sys_sched_getattr - similar to sched_getparam, but with sched_attr
-+ * @pid: the pid in question.
-+ * @uattr: structure containing the extended parameters.
-+ * @size: sizeof(attr) for fwd/bwd comp.
-+ * @flags: for future extension.
-+ */
-+SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
-+ unsigned int, size, unsigned int, flags)
-+{
-+ struct sched_attr attr = {
-+ .size = sizeof(struct sched_attr),
-+ };
-+ struct task_struct *p;
-+ int retval;
-+
-+ if (!uattr || pid < 0 || size > PAGE_SIZE ||
-+ size < SCHED_ATTR_SIZE_VER0 || flags)
-+ return -EINVAL;
-+
-+ rcu_read_lock();
-+ p = find_process_by_pid(pid);
-+ retval = -ESRCH;
-+ if (!p)
-+ goto out_unlock;
-+
-+ retval = security_task_getscheduler(p);
-+ if (retval)
-+ goto out_unlock;
-+
-+ attr.sched_policy = p->policy;
-+ if (p->sched_reset_on_fork)
-+ attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
-+ if (task_has_dl_policy(p))
-+ __getparam_dl(p, &attr);
-+ else if (task_has_rt_policy(p))
-+ attr.sched_priority = p->rt_priority;
-+ else
-+ attr.sched_nice = task_nice(p);
-+
-+ rcu_read_unlock();
-+
-+ retval = sched_read_attr(uattr, &attr, size);
-+ return retval;
-+
-+out_unlock:
-+ rcu_read_unlock();
-+ return retval;
-+}
-+
-+long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
-+{
-+ cpumask_var_t cpus_allowed, new_mask;
-+ struct task_struct *p;
-+ int retval;
-+
-+ rcu_read_lock();
-+
-+ p = find_process_by_pid(pid);
-+ if (!p) {
-+ rcu_read_unlock();
-+ return -ESRCH;
-+ }
-+
-+ /* Prevent p going away */
-+ get_task_struct(p);
-+ rcu_read_unlock();
-+
-+ if (p->flags & PF_NO_SETAFFINITY) {
-+ retval = -EINVAL;
-+ goto out_put_task;
-+ }
-+ if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
-+ retval = -ENOMEM;
-+ goto out_put_task;
-+ }
-+ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
-+ retval = -ENOMEM;
-+ goto out_free_cpus_allowed;
-+ }
-+ retval = -EPERM;
-+ if (!check_same_owner(p)) {
-+ rcu_read_lock();
-+ if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
-+ rcu_read_unlock();
-+ goto out_free_new_mask;
-+ }
-+ rcu_read_unlock();
-+ }
-+
-+ retval = security_task_setscheduler(p);
-+ if (retval)
-+ goto out_free_new_mask;
-+
-+
-+ cpuset_cpus_allowed(p, cpus_allowed);
-+ cpumask_and(new_mask, in_mask, cpus_allowed);
-+
-+ /*
-+ * Since bandwidth control happens on root_domain basis,
-+ * if admission test is enabled, we only admit -deadline
-+ * tasks allowed to run on all the CPUs in the task's
-+ * root_domain.
-+ */
-+#ifdef CONFIG_SMP
-+ if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
-+ rcu_read_lock();
-+ if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) {
-+ retval = -EBUSY;
-+ rcu_read_unlock();
-+ goto out_free_new_mask;
-+ }
-+ rcu_read_unlock();
-+ }
-+#endif
-+again:
-+ retval = set_cpus_allowed_ptr(p, new_mask);
-+
-+ if (!retval) {
-+ cpuset_cpus_allowed(p, cpus_allowed);
-+ if (!cpumask_subset(new_mask, cpus_allowed)) {
-+ /*
-+ * We must have raced with a concurrent cpuset
-+ * update. Just reset the cpus_allowed to the
-+ * cpuset's cpus_allowed
-+ */
-+ cpumask_copy(new_mask, cpus_allowed);
-+ goto again;
-+ }
-+ }
-+out_free_new_mask:
-+ free_cpumask_var(new_mask);
-+out_free_cpus_allowed:
-+ free_cpumask_var(cpus_allowed);
-+out_put_task:
-+ put_task_struct(p);
-+ return retval;
-+}
-+
-+static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
-+ struct cpumask *new_mask)
-+{
-+ if (len < cpumask_size())
-+ cpumask_clear(new_mask);
-+ else if (len > cpumask_size())
-+ len = cpumask_size();
-+
-+ return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
-+}
-+
-+/**
-+ * sys_sched_setaffinity - set the cpu affinity of a process
-+ * @pid: pid of the process
-+ * @len: length in bytes of the bitmask pointed to by user_mask_ptr
-+ * @user_mask_ptr: user-space pointer to the new cpu mask
-+ *
-+ * Return: 0 on success. An error code otherwise.
-+ */
-+SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
-+ unsigned long __user *, user_mask_ptr)
-+{
-+ cpumask_var_t new_mask;
-+ int retval;
-+
-+ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
-+ return -ENOMEM;
-+
-+ retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
-+ if (retval == 0)
-+ retval = sched_setaffinity(pid, new_mask);
-+ free_cpumask_var(new_mask);
-+ return retval;
-+}
-+
-+long sched_getaffinity(pid_t pid, struct cpumask *mask)
-+{
-+ struct task_struct *p;
-+ unsigned long flags;
-+ int retval;
-+
-+ rcu_read_lock();
-+
-+ retval = -ESRCH;
-+ p = find_process_by_pid(pid);
-+ if (!p)
-+ goto out_unlock;
-+
-+ retval = security_task_getscheduler(p);
-+ if (retval)
-+ goto out_unlock;
-+
-+ raw_spin_lock_irqsave(&p->pi_lock, flags);
-+ cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
-+ raw_spin_unlock_irqrestore(&p->pi_lock, flags);
-+
-+out_unlock:
-+ rcu_read_unlock();
-+
-+ return retval;
-+}
-+
-+/**
-+ * sys_sched_getaffinity - get the cpu affinity of a process
-+ * @pid: pid of the process
-+ * @len: length in bytes of the bitmask pointed to by user_mask_ptr
-+ * @user_mask_ptr: user-space pointer to hold the current cpu mask
-+ *
-+ * Return: 0 on success. An error code otherwise.
-+ */
-+SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
-+ unsigned long __user *, user_mask_ptr)
-+{
-+ int ret;
-+ cpumask_var_t mask;
-+
-+ if ((len * BITS_PER_BYTE) < nr_cpu_ids)
-+ return -EINVAL;
-+ if (len & (sizeof(unsigned long)-1))
-+ return -EINVAL;
-+
-+ if (!alloc_cpumask_var(&mask, GFP_KERNEL))
-+ return -ENOMEM;
-+
-+ ret = sched_getaffinity(pid, mask);
-+ if (ret == 0) {
-+ size_t retlen = min_t(size_t, len, cpumask_size());
-+
-+ if (copy_to_user(user_mask_ptr, mask, retlen))
-+ ret = -EFAULT;
-+ else
-+ ret = retlen;
-+ }
-+ free_cpumask_var(mask);
-+
-+ return ret;
-+}
-+
-+/**
-+ * sys_sched_yield - yield the current processor to other threads.
-+ *
-+ * This function yields the current CPU to other tasks. If there are no
-+ * other threads running on this CPU then this function will return.
-+ *
-+ * Return: 0.
-+ */
-+SYSCALL_DEFINE0(sched_yield)
-+{
-+ struct rq *rq = this_rq_lock();
-+
-+ schedstat_inc(rq, yld_count);
-+ current->sched_class->yield_task(rq);
-+
-+ /*
-+ * Since we are going to call schedule() anyway, there's
-+ * no need to preempt or enable interrupts:
-+ */
-+ __release(rq->lock);
-+ spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
-+ do_raw_spin_unlock(&rq->lock);
-+ sched_preempt_enable_no_resched();
-+
-+ schedule();
-+
-+ return 0;
-+}
-+
-+int __sched _cond_resched(void)
-+{
-+ if (should_resched()) {
-+ preempt_schedule_common();
-+ return 1;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(_cond_resched);
-+
-+/*
-+ * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
-+ * call schedule, and on return reacquire the lock.
-+ *
-+ * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
-+ * operations here to prevent schedule() from being called twice (once via
-+ * spin_unlock(), once by hand).
-+ */
-+int __cond_resched_lock(spinlock_t *lock)
-+{
-+ int resched = should_resched();
-+ int ret = 0;
-+
-+ lockdep_assert_held(lock);
-+
-+ if (spin_needbreak(lock) || resched) {
-+ spin_unlock(lock);
-+ if (resched)
-+ preempt_schedule_common();
-+ else
-+ cpu_relax();
-+ ret = 1;
-+ spin_lock(lock);
-+ }
-+ return ret;
-+}
-+EXPORT_SYMBOL(__cond_resched_lock);
-+
-+int __sched __cond_resched_softirq(void)
-+{
-+ BUG_ON(!in_softirq());
-+
-+ if (should_resched()) {
-+ local_bh_enable();
-+ preempt_schedule_common();
-+ local_bh_disable();
-+ return 1;
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(__cond_resched_softirq);
-+
-+/**
-+ * yield - yield the current processor to other threads.
-+ *
-+ * Do not ever use this function, there's a 99% chance you're doing it wrong.
-+ *
-+ * The scheduler is at all times free to pick the calling task as the most
-+ * eligible task to run, if removing the yield() call from your code breaks
-+ * it, its already broken.
-+ *
-+ * Typical broken usage is:
-+ *
-+ * while (!event)
-+ * yield();
-+ *
-+ * where one assumes that yield() will let 'the other' process run that will
-+ * make event true. If the current task is a SCHED_FIFO task that will never
-+ * happen. Never use yield() as a progress guarantee!!
-+ *
-+ * If you want to use yield() to wait for something, use wait_event().
-+ * If you want to use yield() to be 'nice' for others, use cond_resched().
-+ * If you still want to use yield(), do not!
-+ */
-+void __sched yield(void)
-+{
-+ set_current_state(TASK_RUNNING);
-+ sys_sched_yield();
-+}
-+EXPORT_SYMBOL(yield);
-+
-+/**
-+ * yield_to - yield the current processor to another thread in
-+ * your thread group, or accelerate that thread toward the
-+ * processor it's on.
-+ * @p: target task
-+ * @preempt: whether task preemption is allowed or not
-+ *
-+ * It's the caller's job to ensure that the target task struct
-+ * can't go away on us before we can do any checks.
-+ *
-+ * Return:
-+ * true (>0) if we indeed boosted the target task.
-+ * false (0) if we failed to boost the target.
-+ * -ESRCH if there's no task to yield to.
-+ */
-+int __sched yield_to(struct task_struct *p, bool preempt)
-+{
-+ struct task_struct *curr = current;
-+ struct rq *rq, *p_rq;
-+ unsigned long flags;
-+ int yielded = 0;
-+
-+ local_irq_save(flags);
-+ rq = this_rq();
-+
-+again:
-+ p_rq = task_rq(p);
-+ /*
-+ * If we're the only runnable task on the rq and target rq also
-+ * has only one task, there's absolutely no point in yielding.
-+ */
-+ if (rq->nr_running == 1 && p_rq->nr_running == 1) {
-+ yielded = -ESRCH;
-+ goto out_irq;
-+ }
-+
-+ double_rq_lock(rq, p_rq);
-+ if (task_rq(p) != p_rq) {
-+ double_rq_unlock(rq, p_rq);
-+ goto again;
-+ }
-+
-+ if (!curr->sched_class->yield_to_task)
-+ goto out_unlock;
-+
-+ if (curr->sched_class != p->sched_class)
-+ goto out_unlock;
-+
-+ if (task_running(p_rq, p) || p->state)
-+ goto out_unlock;
-+
-+ yielded = curr->sched_class->yield_to_task(rq, p, preempt);
-+ if (yielded) {
-+ schedstat_inc(rq, yld_count);
-+ /*
-+ * Make p's CPU reschedule; pick_next_entity takes care of
-+ * fairness.
-+ */
-+ if (preempt && rq != p_rq)
-+ resched_curr(p_rq);
-+ }
-+
-+out_unlock:
-+ double_rq_unlock(rq, p_rq);
-+out_irq:
-+ local_irq_restore(flags);
-+
-+ if (yielded > 0)
-+ schedule();
-+
-+ return yielded;
-+}
-+EXPORT_SYMBOL_GPL(yield_to);
-+
-+/*
-+ * This task is about to go to sleep on IO. Increment rq->nr_iowait so
-+ * that process accounting knows that this is a task in IO wait state.
-+ */
-+long __sched io_schedule_timeout(long timeout)
-+{
-+ int old_iowait = current->in_iowait;
-+ struct rq *rq;
-+ long ret;
-+
-+ current->in_iowait = 1;
-+ blk_schedule_flush_plug(current);
-+
-+ delayacct_blkio_start();
-+ rq = raw_rq();
-+ atomic_inc(&rq->nr_iowait);
-+ ret = schedule_timeout(timeout);
-+ current->in_iowait = old_iowait;
-+ atomic_dec(&rq->nr_iowait);
-+ delayacct_blkio_end();
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL(io_schedule_timeout);
-+
-+/**
-+ * sys_sched_get_priority_max - return maximum RT priority.
-+ * @policy: scheduling class.
-+ *
-+ * Return: On success, this syscall returns the maximum
-+ * rt_priority that can be used by a given scheduling class.
-+ * On failure, a negative error code is returned.
-+ */
-+SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
-+{
-+ int ret = -EINVAL;
-+
-+ switch (policy) {
-+ case SCHED_FIFO:
-+ case SCHED_RR:
-+ ret = MAX_USER_RT_PRIO-1;
-+ break;
-+ case SCHED_DEADLINE:
-+ case SCHED_NORMAL:
-+ case SCHED_BATCH:
-+ case SCHED_IDLE:
-+ ret = 0;
-+ break;
-+ }
-+ return ret;
-+}
-+
-+/**
-+ * sys_sched_get_priority_min - return minimum RT priority.
-+ * @policy: scheduling class.
-+ *
-+ * Return: On success, this syscall returns the minimum
-+ * rt_priority that can be used by a given scheduling class.
-+ * On failure, a negative error code is returned.
-+ */
-+SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
-+{
-+ int ret = -EINVAL;
-+
-+ switch (policy) {
-+ case SCHED_FIFO:
-+ case SCHED_RR:
-+ ret = 1;
-+ break;
-+ case SCHED_DEADLINE:
-+ case SCHED_NORMAL:
-+ case SCHED_BATCH:
-+ case SCHED_IDLE:
-+ ret = 0;
-+ }
-+ return ret;
-+}
-+
-+/**
-+ * sys_sched_rr_get_interval - return the default timeslice of a process.
-+ * @pid: pid of the process.
-+ * @interval: userspace pointer to the timeslice value.
-+ *
-+ * this syscall writes the default timeslice value of a given process
-+ * into the user-space timespec buffer. A value of '0' means infinity.
-+ *
-+ * Return: On success, 0 and the timeslice is in @interval. Otherwise,
-+ * an error code.
-+ */
-+SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
-+ struct timespec __user *, interval)
-+{
-+ struct task_struct *p;
-+ unsigned int time_slice;
-+ unsigned long flags;
-+ struct rq *rq;
-+ int retval;
-+ struct timespec t;
-+
-+ if (pid < 0)
-+ return -EINVAL;
-+
-+ retval = -ESRCH;
-+ rcu_read_lock();
-+ p = find_process_by_pid(pid);
-+ if (!p)
-+ goto out_unlock;
-+
-+ retval = security_task_getscheduler(p);
-+ if (retval)
-+ goto out_unlock;
-+
-+ rq = task_rq_lock(p, &flags);
-+ time_slice = 0;
-+ if (p->sched_class->get_rr_interval)
-+ time_slice = p->sched_class->get_rr_interval(rq, p);
-+ task_rq_unlock(rq, p, &flags);
-+
-+ rcu_read_unlock();
-+ jiffies_to_timespec(time_slice, &t);
-+ retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
-+ return retval;
-+
-+out_unlock:
-+ rcu_read_unlock();
-+ return retval;
-+}
-+
-+static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
-+
-+void sched_show_task(struct task_struct *p)
-+{
-+ unsigned long free = 0;
-+ int ppid;
-+ unsigned long state = p->state;
-+
-+ if (state)
-+ state = __ffs(state) + 1;
-+ printk(KERN_INFO "%-15.15s %c", p->comm,
-+ state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
-+#if BITS_PER_LONG == 32
-+ if (state == TASK_RUNNING)
-+ printk(KERN_CONT " running ");
-+ else
-+ printk(KERN_CONT " %08lx ", thread_saved_pc(p));
-+#else
-+ if (state == TASK_RUNNING)
-+ printk(KERN_CONT " running task ");
-+ else
-+ printk(KERN_CONT " %016lx ", thread_saved_pc(p));
-+#endif
-+#ifdef CONFIG_DEBUG_STACK_USAGE
-+ free = stack_not_used(p);
-+#endif
-+ ppid = 0;
-+ rcu_read_lock();
-+ if (pid_alive(p))
-+ ppid = task_pid_nr(rcu_dereference(p->real_parent));
-+ rcu_read_unlock();
-+ printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
-+ task_pid_nr(p), ppid,
-+ (unsigned long)task_thread_info(p)->flags);
-+
-+ print_worker_info(KERN_INFO, p);
-+ show_stack(p, NULL);
-+}
-+
-+void show_state_filter(unsigned long state_filter)
-+{
-+ struct task_struct *g, *p;
-+
-+#if BITS_PER_LONG == 32
-+ printk(KERN_INFO
-+ " task PC stack pid father\n");
-+#else
-+ printk(KERN_INFO
-+ " task PC stack pid father\n");
-+#endif
-+ rcu_read_lock();
-+ for_each_process_thread(g, p) {
-+ /*
-+ * reset the NMI-timeout, listing all files on a slow
-+ * console might take a lot of time:
-+ */
-+ touch_nmi_watchdog();
-+ if (!state_filter || (p->state & state_filter))
-+ sched_show_task(p);
-+ }
-+
-+ touch_all_softlockup_watchdogs();
-+
-+#ifdef CONFIG_SCHED_DEBUG
-+ sysrq_sched_debug_show();
-+#endif
-+ rcu_read_unlock();
-+ /*
-+ * Only show locks if all tasks are dumped:
-+ */
-+ if (!state_filter)
-+ debug_show_all_locks();
-+}
-+
-+void init_idle_bootup_task(struct task_struct *idle)
-+{
-+ idle->sched_class = &idle_sched_class;
-+}
-+
-+/**
-+ * init_idle - set up an idle thread for a given CPU
-+ * @idle: task in question
-+ * @cpu: cpu the idle task belongs to
-+ *
-+ * NOTE: this function does not set the idle thread's NEED_RESCHED
-+ * flag, to make booting more robust.
-+ */
-+void init_idle(struct task_struct *idle, int cpu)
-+{
-+ struct rq *rq = cpu_rq(cpu);
-+ unsigned long flags;
-+
-+ raw_spin_lock_irqsave(&rq->lock, flags);
-+
-+ __sched_fork(0, idle);
-+ idle->state = TASK_RUNNING;
-+ idle->se.exec_start = sched_clock();
-+
-+ do_set_cpus_allowed(idle, cpumask_of(cpu));
-+ /*
-+ * We're having a chicken and egg problem, even though we are
-+ * holding rq->lock, the cpu isn't yet set to this cpu so the
-+ * lockdep check in task_group() will fail.
-+ *
-+ * Similar case to sched_fork(). / Alternatively we could
-+ * use task_rq_lock() here and obtain the other rq->lock.
-+ *
-+ * Silence PROVE_RCU
-+ */
-+ rcu_read_lock();
-+ __set_task_cpu(idle, cpu);
-+ rcu_read_unlock();
-+
-+ rq->curr = rq->idle = idle;
-+ idle->on_rq = TASK_ON_RQ_QUEUED;
-+#if defined(CONFIG_SMP)
-+ idle->on_cpu = 1;
-+#endif
-+ raw_spin_unlock_irqrestore(&rq->lock, flags);
-+
-+ /* Set the preempt count _outside_ the spinlocks! */
-+ init_idle_preempt_count(idle, cpu);
-+
-+ /*
-+ * The idle tasks have their own, simple scheduling class:
-+ */
-+ idle->sched_class = &idle_sched_class;
-+ ftrace_graph_init_idle_task(idle, cpu);
-+ vtime_init_idle(idle, cpu);
-+#if defined(CONFIG_SMP)
-+ sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
-+#endif
-+}
-+
-+int cpuset_cpumask_can_shrink(const struct cpumask *cur,
-+ const struct cpumask *trial)
-+{
-+ int ret = 1, trial_cpus;
-+ struct dl_bw *cur_dl_b;
-+ unsigned long flags;
-+
-+ if (!cpumask_weight(cur))
-+ return ret;
-+
-+ rcu_read_lock_sched();
-+ cur_dl_b = dl_bw_of(cpumask_any(cur));
-+ trial_cpus = cpumask_weight(trial);
-+
-+ raw_spin_lock_irqsave(&cur_dl_b->lock, flags);
-+ if (cur_dl_b->bw != -1 &&
-+ cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw)
-+ ret = 0;
-+ raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags);
-+ rcu_read_unlock_sched();
-+
-+ return ret;
-+}
-+
-+int task_can_attach(struct task_struct *p,
-+ const struct cpumask *cs_cpus_allowed)
-+{
-+ int ret = 0;
-+
-+ /*
-+ * Kthreads which disallow setaffinity shouldn't be moved
-+ * to a new cpuset; we don't want to change their cpu
-+ * affinity and isolating such threads by their set of
-+ * allowed nodes is unnecessary. Thus, cpusets are not
-+ * applicable for such threads. This prevents checking for
-+ * success of set_cpus_allowed_ptr() on all attached tasks
-+ * before cpus_allowed may be changed.
-+ */
-+ if (p->flags & PF_NO_SETAFFINITY) {
-+ ret = -EINVAL;
-+ goto out;
-+ }
-+
-+#ifdef CONFIG_SMP
-+ if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
-+ cs_cpus_allowed)) {
-+ unsigned int dest_cpu = cpumask_any_and(cpu_active_mask,
-+ cs_cpus_allowed);
-+ struct dl_bw *dl_b;
-+ bool overflow;
-+ int cpus;
-+ unsigned long flags;
-+
-+ rcu_read_lock_sched();
-+ dl_b = dl_bw_of(dest_cpu);
-+ raw_spin_lock_irqsave(&dl_b->lock, flags);
-+ cpus = dl_bw_cpus(dest_cpu);
-+ overflow = __dl_overflow(dl_b, cpus, 0, p->dl.dl_bw);
-+ if (overflow)
-+ ret = -EBUSY;
-+ else {
-+ /*
-+ * We reserve space for this task in the destination
-+ * root_domain, as we can't fail after this point.
-+ * We will free resources in the source root_domain
-+ * later on (see set_cpus_allowed_dl()).
-+ */
-+ __dl_add(dl_b, p->dl.dl_bw);
-+ }
-+ raw_spin_unlock_irqrestore(&dl_b->lock, flags);
-+ rcu_read_unlock_sched();
-+
-+ }
-+#endif
-+out:
-+ return ret;
-+}
-+
-+#ifdef CONFIG_SMP
-+/*
-+ * move_queued_task - move a queued task to new rq.
-+ *
-+ * Returns (locked) new rq. Old rq's lock is released.
-+ */
-+static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
-+{
-+ struct rq *rq = task_rq(p);
-+
-+ lockdep_assert_held(&rq->lock);
-+
-+ dequeue_task(rq, p, 0);
-+ p->on_rq = TASK_ON_RQ_MIGRATING;
-+ set_task_cpu(p, new_cpu);
-+ raw_spin_unlock(&rq->lock);
-+
-+ rq = cpu_rq(new_cpu);
-+
-+ raw_spin_lock(&rq->lock);
-+ BUG_ON(task_cpu(p) != new_cpu);
-+ p->on_rq = TASK_ON_RQ_QUEUED;
-+ enqueue_task(rq, p, 0);
-+ check_preempt_curr(rq, p, 0);
-+
-+ return rq;
-+}
-+
-+void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
-+{
-+ if (p->sched_class->set_cpus_allowed)
-+ p->sched_class->set_cpus_allowed(p, new_mask);
-+
-+ cpumask_copy(&p->cpus_allowed, new_mask);
-+ p->nr_cpus_allowed = cpumask_weight(new_mask);
-+}
-+
-+/*
-+ * This is how migration works:
-+ *
-+ * 1) we invoke migration_cpu_stop() on the target CPU using
-+ * stop_one_cpu().
-+ * 2) stopper starts to run (implicitly forcing the migrated thread
-+ * off the CPU)
-+ * 3) it checks whether the migrated task is still in the wrong runqueue.
-+ * 4) if it's in the wrong runqueue then the migration thread removes
-+ * it and puts it into the right queue.
-+ * 5) stopper completes and stop_one_cpu() returns and the migration
-+ * is done.
-+ */
-+
-+/*
-+ * Change a given task's CPU affinity. Migrate the thread to a
-+ * proper CPU and schedule it away if the CPU it's executing on
-+ * is removed from the allowed bitmask.
-+ *
-+ * NOTE: the caller must have a valid reference to the task, the
-+ * task must not exit() & deallocate itself prematurely. The
-+ * call is not atomic; no spinlocks may be held.
-+ */
-+int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
-+{
-+ unsigned long flags;
-+ struct rq *rq;
-+ unsigned int dest_cpu;
-+ int ret = 0;
-+
-+ rq = task_rq_lock(p, &flags);
-+
-+ if (cpumask_equal(&p->cpus_allowed, new_mask))
-+ goto out;
-+
-+ if (!cpumask_intersects(new_mask, cpu_active_mask)) {
-+ ret = -EINVAL;
-+ goto out;
-+ }
-+
-+ do_set_cpus_allowed(p, new_mask);
-+
-+ /* Can the task run on the task's current CPU? If so, we're done */
-+ if (cpumask_test_cpu(task_cpu(p), new_mask))
-+ goto out;
-+
-+ dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
-+ if (task_running(rq, p) || p->state == TASK_WAKING) {
-+ struct migration_arg arg = { p, dest_cpu };
-+ /* Need help from migration thread: drop lock and wait. */
-+ task_rq_unlock(rq, p, &flags);
-+ stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
-+ tlb_migrate_finish(p->mm);
-+ return 0;
-+ } else if (task_on_rq_queued(p))
-+ rq = move_queued_task(p, dest_cpu);
-+out:
-+ task_rq_unlock(rq, p, &flags);
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
-+
-+/*
-+ * Move (not current) task off this cpu, onto dest cpu. We're doing
-+ * this because either it can't run here any more (set_cpus_allowed()
-+ * away from this CPU, or CPU going down), or because we're
-+ * attempting to rebalance this task on exec (sched_exec).
-+ *
-+ * So we race with normal scheduler movements, but that's OK, as long
-+ * as the task is no longer on this CPU.
-+ *
-+ * Returns non-zero if task was successfully migrated.
-+ */
-+static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
-+{
-+ struct rq *rq;
-+ int ret = 0;
-+
-+ if (unlikely(!cpu_active(dest_cpu)))
-+ return ret;
-+
-+ rq = cpu_rq(src_cpu);
-+
-+ raw_spin_lock(&p->pi_lock);
-+ raw_spin_lock(&rq->lock);
-+ /* Already moved. */
-+ if (task_cpu(p) != src_cpu)
-+ goto done;
-+
-+ /* Affinity changed (again). */
-+ if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
-+ goto fail;
-+
-+ /*
-+ * If we're not on a rq, the next wake-up will ensure we're
-+ * placed properly.
-+ */
-+ if (task_on_rq_queued(p))
-+ rq = move_queued_task(p, dest_cpu);
-+done:
-+ ret = 1;
-+fail:
-+ raw_spin_unlock(&rq->lock);
-+ raw_spin_unlock(&p->pi_lock);
-+ return ret;
-+}
-+
-+#ifdef CONFIG_NUMA_BALANCING
-+/* Migrate current task p to target_cpu */
-+int migrate_task_to(struct task_struct *p, int target_cpu)
-+{
-+ struct migration_arg arg = { p, target_cpu };
-+ int curr_cpu = task_cpu(p);
-+
-+ if (curr_cpu == target_cpu)
-+ return 0;
-+
-+ if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
-+ return -EINVAL;
-+
-+ /* TODO: This is not properly updating schedstats */
-+
-+ trace_sched_move_numa(p, curr_cpu, target_cpu);
-+ return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
-+}
-+
-+/*
-+ * Requeue a task on a given node and accurately track the number of NUMA
-+ * tasks on the runqueues
-+ */
-+void sched_setnuma(struct task_struct *p, int nid)
-+{
-+ struct rq *rq;
-+ unsigned long flags;
-+ bool queued, running;
-+
-+ rq = task_rq_lock(p, &flags);
-+ queued = task_on_rq_queued(p);
-+ running = task_current(rq, p);
-+
-+ if (queued)
-+ dequeue_task(rq, p, 0);
-+ if (running)
-+ put_prev_task(rq, p);
-+
-+ p->numa_preferred_nid = nid;
-+
-+ if (running)
-+ p->sched_class->set_curr_task(rq);
-+ if (queued)
-+ enqueue_task(rq, p, 0);
-+ task_rq_unlock(rq, p, &flags);
-+}
-+#endif
-+
-+/*
-+ * migration_cpu_stop - this will be executed by a highprio stopper thread
-+ * and performs thread migration by bumping thread off CPU then
-+ * 'pushing' onto another runqueue.
-+ */
-+static int migration_cpu_stop(void *data)
-+{
-+ struct migration_arg *arg = data;
-+
-+ /*
-+ * The original target cpu might have gone down and we might
-+ * be on another cpu but it doesn't matter.
-+ */
-+ local_irq_disable();
-+ /*
-+ * We need to explicitly wake pending tasks before running
-+ * __migrate_task() such that we will not miss enforcing cpus_allowed
-+ * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
-+ */
-+ sched_ttwu_pending();
-+ __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
-+ local_irq_enable();
-+ return 0;
-+}
-+
-+#ifdef CONFIG_HOTPLUG_CPU
-+
-+/*
-+ * Ensures that the idle task is using init_mm right before its cpu goes
-+ * offline.
-+ */
-+void idle_task_exit(void)
-+{
-+ struct mm_struct *mm = current->active_mm;
-+
-+ BUG_ON(cpu_online(smp_processor_id()));
-+
-+ if (mm != &init_mm) {
-+ switch_mm(mm, &init_mm, current);
-+ finish_arch_post_lock_switch();
-+ }
-+ mmdrop(mm);
-+}
-+
-+/*
-+ * Since this CPU is going 'away' for a while, fold any nr_active delta
-+ * we might have. Assumes we're called after migrate_tasks() so that the
-+ * nr_active count is stable.
-+ *
-+ * Also see the comment "Global load-average calculations".
-+ */
-+static void calc_load_migrate(struct rq *rq)
-+{
-+ long delta = calc_load_fold_active(rq);
-+ if (delta)
-+ atomic_long_add(delta, &calc_load_tasks);
-+}
-+
-+static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
-+{
-+}
-+
-+static const struct sched_class fake_sched_class = {
-+ .put_prev_task = put_prev_task_fake,
-+};
-+
-+static struct task_struct fake_task = {
-+ /*
-+ * Avoid pull_{rt,dl}_task()
-+ */
-+ .prio = MAX_PRIO + 1,
-+ .sched_class = &fake_sched_class,
-+};
-+
-+/*
-+ * Migrate all tasks from the rq, sleeping tasks will be migrated by
-+ * try_to_wake_up()->select_task_rq().
-+ *
-+ * Called with rq->lock held even though we'er in stop_machine() and
-+ * there's no concurrency possible, we hold the required locks anyway
-+ * because of lock validation efforts.
-+ */
-+static void migrate_tasks(unsigned int dead_cpu)
-+{
-+ struct rq *rq = cpu_rq(dead_cpu);
-+ struct task_struct *next, *stop = rq->stop;
-+ int dest_cpu;
-+
-+ /*
-+ * Fudge the rq selection such that the below task selection loop
-+ * doesn't get stuck on the currently eligible stop task.
-+ *
-+ * We're currently inside stop_machine() and the rq is either stuck
-+ * in the stop_machine_cpu_stop() loop, or we're executing this code,
-+ * either way we should never end up calling schedule() until we're
-+ * done here.
-+ */
-+ rq->stop = NULL;
-+
-+ /*
-+ * put_prev_task() and pick_next_task() sched
-+ * class method both need to have an up-to-date
-+ * value of rq->clock[_task]
-+ */
-+ update_rq_clock(rq);
-+
-+ for ( ; ; ) {
-+ /*
-+ * There's this thread running, bail when that's the only
-+ * remaining thread.
-+ */
-+ if (rq->nr_running == 1)
-+ break;
-+
-+ next = pick_next_task(rq, &fake_task);
-+ BUG_ON(!next);
-+ next->sched_class->put_prev_task(rq, next);
-+
-+ /* Find suitable destination for @next, with force if needed. */
-+ dest_cpu = select_fallback_rq(dead_cpu, next);
-+ raw_spin_unlock(&rq->lock);
-+
-+ __migrate_task(next, dead_cpu, dest_cpu);
-+
-+ raw_spin_lock(&rq->lock);
-+ }
-+
-+ rq->stop = stop;
-+}
-+
-+#endif /* CONFIG_HOTPLUG_CPU */
-+
-+#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
-+
-+static struct ctl_table sd_ctl_dir[] = {
-+ {
-+ .procname = "sched_domain",
-+ .mode = 0555,
-+ },
-+ {}
-+};
-+
-+static struct ctl_table sd_ctl_root[] = {
-+ {
-+ .procname = "kernel",
-+ .mode = 0555,
-+ .child = sd_ctl_dir,
-+ },
-+ {}
-+};
-+
-+static struct ctl_table *sd_alloc_ctl_entry(int n)
-+{
-+ struct ctl_table *entry =
-+ kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
-+
-+ return entry;
-+}
-+
-+static void sd_free_ctl_entry(struct ctl_table **tablep)
-+{
-+ struct ctl_table *entry;
-+
-+ /*
-+ * In the intermediate directories, both the child directory and
-+ * procname are dynamically allocated and could fail but the mode
-+ * will always be set. In the lowest directory the names are
-+ * static strings and all have proc handlers.
-+ */
-+ for (entry = *tablep; entry->mode; entry++) {
-+ if (entry->child)
-+ sd_free_ctl_entry(&entry->child);
-+ if (entry->proc_handler == NULL)
-+ kfree(entry->procname);
-+ }
-+
-+ kfree(*tablep);
-+ *tablep = NULL;
-+}
-+
-+static int min_load_idx = 0;
-+static int max_load_idx = CPU_LOAD_IDX_MAX-1;
-+
-+static void
-+set_table_entry(struct ctl_table *entry,
-+ const char *procname, void *data, int maxlen,
-+ umode_t mode, proc_handler *proc_handler,
-+ bool load_idx)
-+{
-+ entry->procname = procname;
-+ entry->data = data;
-+ entry->maxlen = maxlen;
-+ entry->mode = mode;
-+ entry->proc_handler = proc_handler;
-+
-+ if (load_idx) {
-+ entry->extra1 = &min_load_idx;
-+ entry->extra2 = &max_load_idx;
-+ }
-+}
-+
-+static struct ctl_table *
-+sd_alloc_ctl_domain_table(struct sched_domain *sd)
-+{
-+ struct ctl_table *table = sd_alloc_ctl_entry(14);
-+
-+ if (table == NULL)
-+ return NULL;
-+
-+ set_table_entry(&table[0], "min_interval", &sd->min_interval,
-+ sizeof(long), 0644, proc_doulongvec_minmax, false);
-+ set_table_entry(&table[1], "max_interval", &sd->max_interval,
-+ sizeof(long), 0644, proc_doulongvec_minmax, false);
-+ set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
-+ sizeof(int), 0644, proc_dointvec_minmax, true);
-+ set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
-+ sizeof(int), 0644, proc_dointvec_minmax, true);
-+ set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
-+ sizeof(int), 0644, proc_dointvec_minmax, true);
-+ set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
-+ sizeof(int), 0644, proc_dointvec_minmax, true);
-+ set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
-+ sizeof(int), 0644, proc_dointvec_minmax, true);
-+ set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
-+ sizeof(int), 0644, proc_dointvec_minmax, false);
-+ set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
-+ sizeof(int), 0644, proc_dointvec_minmax, false);
-+ set_table_entry(&table[9], "cache_nice_tries",
-+ &sd->cache_nice_tries,
-+ sizeof(int), 0644, proc_dointvec_minmax, false);
-+ set_table_entry(&table[10], "flags", &sd->flags,
-+ sizeof(int), 0644, proc_dointvec_minmax, false);
-+ set_table_entry(&table[11], "max_newidle_lb_cost",
-+ &sd->max_newidle_lb_cost,
-+ sizeof(long), 0644, proc_doulongvec_minmax, false);
-+ set_table_entry(&table[12], "name", sd->name,
-+ CORENAME_MAX_SIZE, 0444, proc_dostring, false);
-+ /* &table[13] is terminator */
-+
-+ return table;
-+}
-+
-+static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
-+{
-+ struct ctl_table *entry, *table;
-+ struct sched_domain *sd;
-+ int domain_num = 0, i;
-+ char buf[32];
-+
-+ for_each_domain(cpu, sd)
-+ domain_num++;
-+ entry = table = sd_alloc_ctl_entry(domain_num + 1);
-+ if (table == NULL)
-+ return NULL;
-+
-+ i = 0;
-+ for_each_domain(cpu, sd) {
-+ snprintf(buf, 32, "domain%d", i);
-+ entry->procname = kstrdup(buf, GFP_KERNEL);
-+ entry->mode = 0555;
-+ entry->child = sd_alloc_ctl_domain_table(sd);
-+ entry++;
-+ i++;
-+ }
-+ return table;
-+}
-+
-+static struct ctl_table_header *sd_sysctl_header;
-+static void register_sched_domain_sysctl(void)
-+{
-+ int i, cpu_num = num_possible_cpus();
-+ struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
-+ char buf[32];
-+
-+ WARN_ON(sd_ctl_dir[0].child);
-+ sd_ctl_dir[0].child = entry;
-+
-+ if (entry == NULL)
-+ return;
-+
-+ for_each_possible_cpu(i) {
-+ snprintf(buf, 32, "cpu%d", i);
-+ entry->procname = kstrdup(buf, GFP_KERNEL);
-+ entry->mode = 0555;
-+ entry->child = sd_alloc_ctl_cpu_table(i);
-+ entry++;
-+ }
-+
-+ WARN_ON(sd_sysctl_header);
-+ sd_sysctl_header = register_sysctl_table(sd_ctl_root);
-+}
-+
-+/* may be called multiple times per register */
-+static void unregister_sched_domain_sysctl(void)
-+{
-+ if (sd_sysctl_header)
-+ unregister_sysctl_table(sd_sysctl_header);
-+ sd_sysctl_header = NULL;
-+ if (sd_ctl_dir[0].child)
-+ sd_free_ctl_entry(&sd_ctl_dir[0].child);
-+}
-+#else
-+static void register_sched_domain_sysctl(void)
-+{
-+}
-+static void unregister_sched_domain_sysctl(void)
-+{
-+}
-+#endif
-+
-+static void set_rq_online(struct rq *rq)
-+{
-+ if (!rq->online) {
-+ const struct sched_class *class;
-+
-+ cpumask_set_cpu(rq->cpu, rq->rd->online);
-+ rq->online = 1;
-+
-+ for_each_class(class) {
-+ if (class->rq_online)
-+ class->rq_online(rq);
-+ }
-+ }
-+}
-+
-+static void set_rq_offline(struct rq *rq)
-+{
-+ if (rq->online) {
-+ const struct sched_class *class;
-+
-+ for_each_class(class) {
-+ if (class->rq_offline)
-+ class->rq_offline(rq);
-+ }
-+
-+ cpumask_clear_cpu(rq->cpu, rq->rd->online);
-+ rq->online = 0;
-+ }
-+}
-+
-+/*
-+ * migration_call - callback that gets triggered when a CPU is added.
-+ * Here we can start up the necessary migration thread for the new CPU.
-+ */
-+static int
-+migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
-+{
-+ int cpu = (long)hcpu;
-+ unsigned long flags;
-+ struct rq *rq = cpu_rq(cpu);
-+
-+ switch (action & ~CPU_TASKS_FROZEN) {
-+
-+ case CPU_UP_PREPARE:
-+ rq->calc_load_update = calc_load_update;
-+ break;
-+
-+ case CPU_ONLINE:
-+ /* Update our root-domain */
-+ raw_spin_lock_irqsave(&rq->lock, flags);
-+ if (rq->rd) {
-+ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
-+
-+ set_rq_online(rq);
-+ }
-+ raw_spin_unlock_irqrestore(&rq->lock, flags);
-+ break;
-+
-+#ifdef CONFIG_HOTPLUG_CPU
-+ case CPU_DYING:
-+ sched_ttwu_pending();
-+ /* Update our root-domain */
-+ raw_spin_lock_irqsave(&rq->lock, flags);
-+ if (rq->rd) {
-+ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
-+ set_rq_offline(rq);
-+ }
-+ migrate_tasks(cpu);
-+ BUG_ON(rq->nr_running != 1); /* the migration thread */
-+ raw_spin_unlock_irqrestore(&rq->lock, flags);
-+ break;
-+
-+ case CPU_DEAD:
-+ calc_load_migrate(rq);
-+ break;
-+#endif
-+ }
-+
-+ update_max_interval();
-+
-+ return NOTIFY_OK;
-+}
-+
-+/*
-+ * Register at high priority so that task migration (migrate_all_tasks)
-+ * happens before everything else. This has to be lower priority than
-+ * the notifier in the perf_event subsystem, though.
-+ */
-+static struct notifier_block migration_notifier = {
-+ .notifier_call = migration_call,
-+ .priority = CPU_PRI_MIGRATION,
-+};
-+
-+static void __cpuinit set_cpu_rq_start_time(void)
-+{
-+ int cpu = smp_processor_id();
-+ struct rq *rq = cpu_rq(cpu);
-+ rq->age_stamp = sched_clock_cpu(cpu);
-+}
-+
-+static int sched_cpu_active(struct notifier_block *nfb,
-+ unsigned long action, void *hcpu)
-+{
-+ switch (action & ~CPU_TASKS_FROZEN) {
-+ case CPU_STARTING:
-+ set_cpu_rq_start_time();
-+ return NOTIFY_OK;
-+ case CPU_ONLINE:
-+ /*
-+ * At this point a starting CPU has marked itself as online via
-+ * set_cpu_online(). But it might not yet have marked itself
-+ * as active, which is essential from here on.
-+ *
-+ * Thus, fall-through and help the starting CPU along.
-+ */
-+ case CPU_DOWN_FAILED:
-+ set_cpu_active((long)hcpu, true);
-+ return NOTIFY_OK;
-+ default:
-+ return NOTIFY_DONE;
-+ }
-+}
-+
-+static int sched_cpu_inactive(struct notifier_block *nfb,
-+ unsigned long action, void *hcpu)
-+{
-+ switch (action & ~CPU_TASKS_FROZEN) {
-+ case CPU_DOWN_PREPARE:
-+ set_cpu_active((long)hcpu, false);
-+ return NOTIFY_OK;
-+ default:
-+ return NOTIFY_DONE;
-+ }
-+}
-+
-+static int __init migration_init(void)
-+{
-+ void *cpu = (void *)(long)smp_processor_id();
-+ int err;
-+
-+ /* Initialize migration for the boot CPU */
-+ err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
-+ BUG_ON(err == NOTIFY_BAD);
-+ migration_call(&migration_notifier, CPU_ONLINE, cpu);
-+ register_cpu_notifier(&migration_notifier);
-+
-+ /* Register cpu active notifiers */
-+ cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
-+ cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
-+
-+ return 0;
-+}
-+early_initcall(migration_init);
-+#endif
-+
-+#ifdef CONFIG_SMP
-+
-+static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */
-+
-+#ifdef CONFIG_SCHED_DEBUG
-+
-+static __read_mostly int sched_debug_enabled;
-+
-+static int __init sched_debug_setup(char *str)
-+{
-+ sched_debug_enabled = 1;
-+
-+ return 0;
-+}
-+early_param("sched_debug", sched_debug_setup);
-+
-+static inline bool sched_debug(void)
-+{
-+ return sched_debug_enabled;
-+}
-+
-+static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
-+ struct cpumask *groupmask)
-+{
-+ struct sched_group *group = sd->groups;
-+
-+ cpumask_clear(groupmask);
-+
-+ printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
-+
-+ if (!(sd->flags & SD_LOAD_BALANCE)) {
-+ printk("does not load-balance\n");
-+ if (sd->parent)
-+ printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
-+ " has parent");
-+ return -1;
-+ }
-+
-+ printk(KERN_CONT "span %*pbl level %s\n",
-+ cpumask_pr_args(sched_domain_span(sd)), sd->name);
-+
-+ if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
-+ printk(KERN_ERR "ERROR: domain->span does not contain "
-+ "CPU%d\n", cpu);
-+ }
-+ if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
-+ printk(KERN_ERR "ERROR: domain->groups does not contain"
-+ " CPU%d\n", cpu);
-+ }
-+
-+ printk(KERN_DEBUG "%*s groups:", level + 1, "");
-+ do {
-+ if (!group) {
-+ printk("\n");
-+ printk(KERN_ERR "ERROR: group is NULL\n");
-+ break;
-+ }
-+
-+ if (!cpumask_weight(sched_group_cpus(group))) {
-+ printk(KERN_CONT "\n");
-+ printk(KERN_ERR "ERROR: empty group\n");
-+ break;
-+ }
-+
-+ if (!(sd->flags & SD_OVERLAP) &&
-+ cpumask_intersects(groupmask, sched_group_cpus(group))) {
-+ printk(KERN_CONT "\n");
-+ printk(KERN_ERR "ERROR: repeated CPUs\n");
-+ break;
-+ }
-+
-+ cpumask_or(groupmask, groupmask, sched_group_cpus(group));
-+
-+ printk(KERN_CONT " %*pbl",
-+ cpumask_pr_args(sched_group_cpus(group)));
-+ if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
-+ printk(KERN_CONT " (cpu_capacity = %d)",
-+ group->sgc->capacity);
-+ }
-+
-+ group = group->next;
-+ } while (group != sd->groups);
-+ printk(KERN_CONT "\n");
-+
-+ if (!cpumask_equal(sched_domain_span(sd), groupmask))
-+ printk(KERN_ERR "ERROR: groups don't span domain->span\n");
-+
-+ if (sd->parent &&
-+ !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
-+ printk(KERN_ERR "ERROR: parent span is not a superset "
-+ "of domain->span\n");
-+ return 0;
-+}
-+
-+static void sched_domain_debug(struct sched_domain *sd, int cpu)
-+{
-+ int level = 0;
-+
-+ if (!sched_debug_enabled)
-+ return;
-+
-+ if (!sd) {
-+ printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
-+ return;
-+ }
-+
-+ printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
-+
-+ for (;;) {
-+ if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
-+ break;
-+ level++;
-+ sd = sd->parent;
-+ if (!sd)
-+ break;
-+ }
-+}
-+#else /* !CONFIG_SCHED_DEBUG */
-+# define sched_domain_debug(sd, cpu) do { } while (0)
-+static inline bool sched_debug(void)
-+{
-+ return false;
-+}
-+#endif /* CONFIG_SCHED_DEBUG */
-+
-+static int sd_degenerate(struct sched_domain *sd)
-+{
-+ if (cpumask_weight(sched_domain_span(sd)) == 1)
-+ return 1;
-+
-+ /* Following flags need at least 2 groups */
-+ if (sd->flags & (SD_LOAD_BALANCE |
-+ SD_BALANCE_NEWIDLE |
-+ SD_BALANCE_FORK |
-+ SD_BALANCE_EXEC |
-+ SD_SHARE_CPUCAPACITY |
-+ SD_SHARE_PKG_RESOURCES |
-+ SD_SHARE_POWERDOMAIN)) {
-+ if (sd->groups != sd->groups->next)
-+ return 0;
-+ }
-+
-+ /* Following flags don't use groups */
-+ if (sd->flags & (SD_WAKE_AFFINE))
-+ return 0;
-+
-+ return 1;
-+}
-+
-+static int
-+sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
-+{
-+ unsigned long cflags = sd->flags, pflags = parent->flags;
-+
-+ if (sd_degenerate(parent))
-+ return 1;
-+
-+ if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
-+ return 0;
-+
-+ /* Flags needing groups don't count if only 1 group in parent */
-+ if (parent->groups == parent->groups->next) {
-+ pflags &= ~(SD_LOAD_BALANCE |
-+ SD_BALANCE_NEWIDLE |
-+ SD_BALANCE_FORK |
-+ SD_BALANCE_EXEC |
-+ SD_SHARE_CPUCAPACITY |
-+ SD_SHARE_PKG_RESOURCES |
-+ SD_PREFER_SIBLING |
-+ SD_SHARE_POWERDOMAIN);
-+ if (nr_node_ids == 1)
-+ pflags &= ~SD_SERIALIZE;
-+ }
-+ if (~cflags & pflags)
-+ return 0;
-+
-+ return 1;
-+}
-+
-+static void free_rootdomain(struct rcu_head *rcu)
-+{
-+ struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
-+
-+ cpupri_cleanup(&rd->cpupri);
-+ cpudl_cleanup(&rd->cpudl);
-+ free_cpumask_var(rd->dlo_mask);
-+ free_cpumask_var(rd->rto_mask);
-+ free_cpumask_var(rd->online);
-+ free_cpumask_var(rd->span);
-+ kfree(rd);
-+}
-+
-+static void rq_attach_root(struct rq *rq, struct root_domain *rd)
-+{
-+ struct root_domain *old_rd = NULL;
-+ unsigned long flags;
-+
-+ raw_spin_lock_irqsave(&rq->lock, flags);
-+
-+ if (rq->rd) {
-+ old_rd = rq->rd;
-+
-+ if (cpumask_test_cpu(rq->cpu, old_rd->online))
-+ set_rq_offline(rq);
-+
-+ cpumask_clear_cpu(rq->cpu, old_rd->span);
-+
-+ /*
-+ * If we dont want to free the old_rd yet then
-+ * set old_rd to NULL to skip the freeing later
-+ * in this function:
-+ */
-+ if (!atomic_dec_and_test(&old_rd->refcount))
-+ old_rd = NULL;
-+ }
-+
-+ atomic_inc(&rd->refcount);
-+ rq->rd = rd;
-+
-+ cpumask_set_cpu(rq->cpu, rd->span);
-+ if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
-+ set_rq_online(rq);
-+
-+ raw_spin_unlock_irqrestore(&rq->lock, flags);
-+
-+ if (old_rd)
-+ call_rcu_sched(&old_rd->rcu, free_rootdomain);
-+}
-+
-+static int init_rootdomain(struct root_domain *rd)
-+{
-+ memset(rd, 0, sizeof(*rd));
-+
-+ if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
-+ goto out;
-+ if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
-+ goto free_span;
-+ if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
-+ goto free_online;
-+ if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
-+ goto free_dlo_mask;
-+
-+ init_dl_bw(&rd->dl_bw);
-+ if (cpudl_init(&rd->cpudl) != 0)
-+ goto free_dlo_mask;
-+
-+ if (cpupri_init(&rd->cpupri) != 0)
-+ goto free_rto_mask;
-+ return 0;
-+
-+free_rto_mask:
-+ free_cpumask_var(rd->rto_mask);
-+free_dlo_mask:
-+ free_cpumask_var(rd->dlo_mask);
-+free_online:
-+ free_cpumask_var(rd->online);
-+free_span:
-+ free_cpumask_var(rd->span);
-+out:
-+ return -ENOMEM;
-+}
-+
-+/*
-+ * By default the system creates a single root-domain with all cpus as
-+ * members (mimicking the global state we have today).
-+ */
-+struct root_domain def_root_domain;
-+
-+static void init_defrootdomain(void)
-+{
-+ init_rootdomain(&def_root_domain);
-+
-+ atomic_set(&def_root_domain.refcount, 1);
-+}
-+
-+static struct root_domain *alloc_rootdomain(void)
-+{
-+ struct root_domain *rd;
-+
-+ rd = kmalloc(sizeof(*rd), GFP_KERNEL);
-+ if (!rd)
-+ return NULL;
-+
-+ if (init_rootdomain(rd) != 0) {
-+ kfree(rd);
-+ return NULL;
-+ }
-+
-+ return rd;
-+}
-+
-+static void free_sched_groups(struct sched_group *sg, int free_sgc)
-+{
-+ struct sched_group *tmp, *first;
-+
-+ if (!sg)
-+ return;
-+
-+ first = sg;
-+ do {
-+ tmp = sg->next;
-+
-+ if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
-+ kfree(sg->sgc);
-+
-+ kfree(sg);
-+ sg = tmp;
-+ } while (sg != first);
-+}
-+
-+static void free_sched_domain(struct rcu_head *rcu)
-+{
-+ struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
-+
-+ /*
-+ * If its an overlapping domain it has private groups, iterate and
-+ * nuke them all.
-+ */
-+ if (sd->flags & SD_OVERLAP) {
-+ free_sched_groups(sd->groups, 1);
-+ } else if (atomic_dec_and_test(&sd->groups->ref)) {
-+ kfree(sd->groups->sgc);
-+ kfree(sd->groups);
-+ }
-+ kfree(sd);
-+}
-+
-+static void destroy_sched_domain(struct sched_domain *sd, int cpu)
-+{
-+ call_rcu(&sd->rcu, free_sched_domain);
-+}
-+
-+static void destroy_sched_domains(struct sched_domain *sd, int cpu)
-+{
-+ for (; sd; sd = sd->parent)
-+ destroy_sched_domain(sd, cpu);
-+}
-+
-+/*
-+ * Keep a special pointer to the highest sched_domain that has
-+ * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
-+ * allows us to avoid some pointer chasing select_idle_sibling().
-+ *
-+ * Also keep a unique ID per domain (we use the first cpu number in
-+ * the cpumask of the domain), this allows us to quickly tell if
-+ * two cpus are in the same cache domain, see cpus_share_cache().
-+ */
-+DEFINE_PER_CPU(struct sched_domain *, sd_llc);
-+DEFINE_PER_CPU(int, sd_llc_size);
-+DEFINE_PER_CPU(int, sd_llc_id);
-+DEFINE_PER_CPU(struct sched_domain *, sd_numa);
-+DEFINE_PER_CPU(struct sched_domain *, sd_busy);
-+DEFINE_PER_CPU(struct sched_domain *, sd_asym);
-+
-+static void update_top_cache_domain(int cpu)
-+{
-+ struct sched_domain *sd;
-+ struct sched_domain *busy_sd = NULL;
-+ int id = cpu;
-+ int size = 1;
-+
-+ sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
-+ if (sd) {
-+ id = cpumask_first(sched_domain_span(sd));
-+ size = cpumask_weight(sched_domain_span(sd));
-+ busy_sd = sd->parent; /* sd_busy */
-+ }
-+ rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd);
-+
-+ rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
-+ per_cpu(sd_llc_size, cpu) = size;
-+ per_cpu(sd_llc_id, cpu) = id;
-+
-+ sd = lowest_flag_domain(cpu, SD_NUMA);
-+ rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
-+
-+ sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
-+ rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
-+}
-+
-+/*
-+ * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
-+ * hold the hotplug lock.
-+ */
-+static void
-+cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
-+{
-+ struct rq *rq = cpu_rq(cpu);
-+ struct sched_domain *tmp;
-+
-+ /* Remove the sched domains which do not contribute to scheduling. */
-+ for (tmp = sd; tmp; ) {
-+ struct sched_domain *parent = tmp->parent;
-+ if (!parent)
-+ break;
-+
-+ if (sd_parent_degenerate(tmp, parent)) {
-+ tmp->parent = parent->parent;
-+ if (parent->parent)
-+ parent->parent->child = tmp;
-+ /*
-+ * Transfer SD_PREFER_SIBLING down in case of a
-+ * degenerate parent; the spans match for this
-+ * so the property transfers.
-+ */
-+ if (parent->flags & SD_PREFER_SIBLING)
-+ tmp->flags |= SD_PREFER_SIBLING;
-+ destroy_sched_domain(parent, cpu);
-+ } else
-+ tmp = tmp->parent;
-+ }
-+
-+ if (sd && sd_degenerate(sd)) {
-+ tmp = sd;
-+ sd = sd->parent;
-+ destroy_sched_domain(tmp, cpu);
-+ if (sd)
-+ sd->child = NULL;
-+ }
-+
-+ sched_domain_debug(sd, cpu);
-+
-+ rq_attach_root(rq, rd);
-+ tmp = rq->sd;
-+ rcu_assign_pointer(rq->sd, sd);
-+ destroy_sched_domains(tmp, cpu);
-+
-+ update_top_cache_domain(cpu);
-+}
-+
-+/* Setup the mask of cpus configured for isolated domains */
-+static int __init isolated_cpu_setup(char *str)
-+{
-+ alloc_bootmem_cpumask_var(&cpu_isolated_map);
-+ cpulist_parse(str, cpu_isolated_map);
-+ return 1;
-+}
-+
-+__setup("isolcpus=", isolated_cpu_setup);
-+
-+struct s_data {
-+ struct sched_domain ** __percpu sd;
-+ struct root_domain *rd;
-+};
-+
-+enum s_alloc {
-+ sa_rootdomain,
-+ sa_sd,
-+ sa_sd_storage,
-+ sa_none,
-+};
-+
-+/*
-+ * Build an iteration mask that can exclude certain CPUs from the upwards
-+ * domain traversal.
-+ *
-+ * Asymmetric node setups can result in situations where the domain tree is of
-+ * unequal depth, make sure to skip domains that already cover the entire
-+ * range.
-+ *
-+ * In that case build_sched_domains() will have terminated the iteration early
-+ * and our sibling sd spans will be empty. Domains should always include the
-+ * cpu they're built on, so check that.
-+ *
-+ */
-+static void build_group_mask(struct sched_domain *sd, struct sched_group *sg)
-+{
-+ const struct cpumask *span = sched_domain_span(sd);
-+ struct sd_data *sdd = sd->private;
-+ struct sched_domain *sibling;
-+ int i;
-+
-+ for_each_cpu(i, span) {
-+ sibling = *per_cpu_ptr(sdd->sd, i);
-+ if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
-+ continue;
-+
-+ cpumask_set_cpu(i, sched_group_mask(sg));
-+ }
-+}
-+
-+/*
-+ * Return the canonical balance cpu for this group, this is the first cpu
-+ * of this group that's also in the iteration mask.
-+ */
-+int group_balance_cpu(struct sched_group *sg)
-+{
-+ return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg));
-+}
-+
-+static int
-+build_overlap_sched_groups(struct sched_domain *sd, int cpu)
-+{
-+ struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
-+ const struct cpumask *span = sched_domain_span(sd);
-+ struct cpumask *covered = sched_domains_tmpmask;
-+ struct sd_data *sdd = sd->private;
-+ struct sched_domain *sibling;
-+ int i;
-+
-+ cpumask_clear(covered);
-+
-+ for_each_cpu(i, span) {
-+ struct cpumask *sg_span;
-+
-+ if (cpumask_test_cpu(i, covered))
-+ continue;
-+
-+ sibling = *per_cpu_ptr(sdd->sd, i);
-+
-+ /* See the comment near build_group_mask(). */
-+ if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
-+ continue;
-+
-+ sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
-+ GFP_KERNEL, cpu_to_node(cpu));
-+
-+ if (!sg)
-+ goto fail;
-+
-+ sg_span = sched_group_cpus(sg);
-+ if (sibling->child)
-+ cpumask_copy(sg_span, sched_domain_span(sibling->child));
-+ else
-+ cpumask_set_cpu(i, sg_span);
-+
-+ cpumask_or(covered, covered, sg_span);
-+
-+ sg->sgc = *per_cpu_ptr(sdd->sgc, i);
-+ if (atomic_inc_return(&sg->sgc->ref) == 1)
-+ build_group_mask(sd, sg);
-+
-+ /*
-+ * Initialize sgc->capacity such that even if we mess up the
-+ * domains and no possible iteration will get us here, we won't
-+ * die on a /0 trap.
-+ */
-+ sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
-+
-+ /*
-+ * Make sure the first group of this domain contains the
-+ * canonical balance cpu. Otherwise the sched_domain iteration
-+ * breaks. See update_sg_lb_stats().
-+ */
-+ if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
-+ group_balance_cpu(sg) == cpu)
-+ groups = sg;
-+
-+ if (!first)
-+ first = sg;
-+ if (last)
-+ last->next = sg;
-+ last = sg;
-+ last->next = first;
-+ }
-+ sd->groups = groups;
-+
-+ return 0;
-+
-+fail:
-+ free_sched_groups(first, 0);
-+
-+ return -ENOMEM;
-+}
-+
-+static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
-+{
-+ struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
-+ struct sched_domain *child = sd->child;
-+
-+ if (child)
-+ cpu = cpumask_first(sched_domain_span(child));
-+
-+ if (sg) {
-+ *sg = *per_cpu_ptr(sdd->sg, cpu);
-+ (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu);
-+ atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */
-+ }
-+
-+ return cpu;
-+}
-+
-+/*
-+ * build_sched_groups will build a circular linked list of the groups
-+ * covered by the given span, and will set each group's ->cpumask correctly,
-+ * and ->cpu_capacity to 0.
-+ *
-+ * Assumes the sched_domain tree is fully constructed
-+ */
-+static int
-+build_sched_groups(struct sched_domain *sd, int cpu)
-+{
-+ struct sched_group *first = NULL, *last = NULL;
-+ struct sd_data *sdd = sd->private;
-+ const struct cpumask *span = sched_domain_span(sd);
-+ struct cpumask *covered;
-+ int i;
-+
-+ get_group(cpu, sdd, &sd->groups);
-+ atomic_inc(&sd->groups->ref);
-+
-+ if (cpu != cpumask_first(span))
-+ return 0;
-+
-+ lockdep_assert_held(&sched_domains_mutex);
-+ covered = sched_domains_tmpmask;
-+
-+ cpumask_clear(covered);
-+
-+ for_each_cpu(i, span) {
-+ struct sched_group *sg;
-+ int group, j;
-+
-+ if (cpumask_test_cpu(i, covered))
-+ continue;
-+
-+ group = get_group(i, sdd, &sg);
-+ cpumask_setall(sched_group_mask(sg));
-+
-+ for_each_cpu(j, span) {
-+ if (get_group(j, sdd, NULL) != group)
-+ continue;
-+
-+ cpumask_set_cpu(j, covered);
-+ cpumask_set_cpu(j, sched_group_cpus(sg));
-+ }
-+
-+ if (!first)
-+ first = sg;
-+ if (last)
-+ last->next = sg;
-+ last = sg;
-+ }
-+ last->next = first;
-+
-+ return 0;
-+}
-+
-+/*
-+ * Initialize sched groups cpu_capacity.
-+ *
-+ * cpu_capacity indicates the capacity of sched group, which is used while
-+ * distributing the load between different sched groups in a sched domain.
-+ * Typically cpu_capacity for all the groups in a sched domain will be same
-+ * unless there are asymmetries in the topology. If there are asymmetries,
-+ * group having more cpu_capacity will pickup more load compared to the
-+ * group having less cpu_capacity.
-+ */
-+static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
-+{
-+ struct sched_group *sg = sd->groups;
-+
-+ WARN_ON(!sg);
-+
-+ do {
-+ sg->group_weight = cpumask_weight(sched_group_cpus(sg));
-+ sg = sg->next;
-+ } while (sg != sd->groups);
-+
-+ if (cpu != group_balance_cpu(sg))
-+ return;
-+
-+ update_group_capacity(sd, cpu);
-+ atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight);
-+}
-+
-+/*
-+ * Initializers for schedule domains
-+ * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
-+ */
-+
-+static int default_relax_domain_level = -1;
-+int sched_domain_level_max;
-+
-+static int __init setup_relax_domain_level(char *str)
-+{
-+ if (kstrtoint(str, 0, &default_relax_domain_level))
-+ pr_warn("Unable to set relax_domain_level\n");
-+
-+ return 1;
-+}
-+__setup("relax_domain_level=", setup_relax_domain_level);
-+
-+static void set_domain_attribute(struct sched_domain *sd,
-+ struct sched_domain_attr *attr)
-+{
-+ int request;
-+
-+ if (!attr || attr->relax_domain_level < 0) {
-+ if (default_relax_domain_level < 0)
-+ return;
-+ else
-+ request = default_relax_domain_level;
-+ } else
-+ request = attr->relax_domain_level;
-+ if (request < sd->level) {
-+ /* turn off idle balance on this domain */
-+ sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
-+ } else {
-+ /* turn on idle balance on this domain */
-+ sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
-+ }
-+}
-+
-+static void __sdt_free(const struct cpumask *cpu_map);
-+static int __sdt_alloc(const struct cpumask *cpu_map);
-+
-+static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
-+ const struct cpumask *cpu_map)
-+{
-+ switch (what) {
-+ case sa_rootdomain:
-+ if (!atomic_read(&d->rd->refcount))
-+ free_rootdomain(&d->rd->rcu); /* fall through */
-+ case sa_sd:
-+ free_percpu(d->sd); /* fall through */
-+ case sa_sd_storage:
-+ __sdt_free(cpu_map); /* fall through */
-+ case sa_none:
-+ break;
-+ }
-+}
-+
-+static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
-+ const struct cpumask *cpu_map)
-+{
-+ memset(d, 0, sizeof(*d));
-+
-+ if (__sdt_alloc(cpu_map))
-+ return sa_sd_storage;
-+ d->sd = alloc_percpu(struct sched_domain *);
-+ if (!d->sd)
-+ return sa_sd_storage;
-+ d->rd = alloc_rootdomain();
-+ if (!d->rd)
-+ return sa_sd;
-+ return sa_rootdomain;
-+}
-+
-+/*
-+ * NULL the sd_data elements we've used to build the sched_domain and
-+ * sched_group structure so that the subsequent __free_domain_allocs()
-+ * will not free the data we're using.
-+ */
-+static void claim_allocations(int cpu, struct sched_domain *sd)
-+{
-+ struct sd_data *sdd = sd->private;
-+
-+ WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
-+ *per_cpu_ptr(sdd->sd, cpu) = NULL;
-+
-+ if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
-+ *per_cpu_ptr(sdd->sg, cpu) = NULL;
-+
-+ if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
-+ *per_cpu_ptr(sdd->sgc, cpu) = NULL;
-+}
-+
-+#ifdef CONFIG_NUMA
-+static int sched_domains_numa_levels;
-+enum numa_topology_type sched_numa_topology_type;
-+static int *sched_domains_numa_distance;
-+int sched_max_numa_distance;
-+static struct cpumask ***sched_domains_numa_masks;
-+static int sched_domains_curr_level;
-+#endif
-+
-+/*
-+ * SD_flags allowed in topology descriptions.
-+ *
-+ * SD_SHARE_CPUCAPACITY - describes SMT topologies
-+ * SD_SHARE_PKG_RESOURCES - describes shared caches
-+ * SD_NUMA - describes NUMA topologies
-+ * SD_SHARE_POWERDOMAIN - describes shared power domain
-+ *
-+ * Odd one out:
-+ * SD_ASYM_PACKING - describes SMT quirks
-+ */
-+#define TOPOLOGY_SD_FLAGS \
-+ (SD_SHARE_CPUCAPACITY | \
-+ SD_SHARE_PKG_RESOURCES | \
-+ SD_NUMA | \
-+ SD_ASYM_PACKING | \
-+ SD_SHARE_POWERDOMAIN)
-+
-+static struct sched_domain *
-+sd_init(struct sched_domain_topology_level *tl, int cpu)
-+{
-+ struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);
-+ int sd_weight, sd_flags = 0;
-+
-+#ifdef CONFIG_NUMA
-+ /*
-+ * Ugly hack to pass state to sd_numa_mask()...
-+ */
-+ sched_domains_curr_level = tl->numa_level;
-+#endif
-+
-+ sd_weight = cpumask_weight(tl->mask(cpu));
-+
-+ if (tl->sd_flags)
-+ sd_flags = (*tl->sd_flags)();
-+ if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
-+ "wrong sd_flags in topology description\n"))
-+ sd_flags &= ~TOPOLOGY_SD_FLAGS;
-+
-+ *sd = (struct sched_domain){
-+ .min_interval = sd_weight,
-+ .max_interval = 2*sd_weight,
-+ .busy_factor = 32,
-+ .imbalance_pct = 125,
-+
-+ .cache_nice_tries = 0,
-+ .busy_idx = 0,
-+ .idle_idx = 0,
-+ .newidle_idx = 0,
-+ .wake_idx = 0,
-+ .forkexec_idx = 0,
-+
-+ .flags = 1*SD_LOAD_BALANCE
-+ | 1*SD_BALANCE_NEWIDLE
-+ | 1*SD_BALANCE_EXEC
-+ | 1*SD_BALANCE_FORK
-+ | 0*SD_BALANCE_WAKE
-+ | 1*SD_WAKE_AFFINE
-+ | 0*SD_SHARE_CPUCAPACITY
-+ | 0*SD_SHARE_PKG_RESOURCES
-+ | 0*SD_SERIALIZE
-+ | 0*SD_PREFER_SIBLING
-+ | 0*SD_NUMA
-+ | sd_flags
-+ ,
-+
-+ .last_balance = jiffies,
-+ .balance_interval = sd_weight,
-+ .smt_gain = 0,
-+ .max_newidle_lb_cost = 0,
-+ .next_decay_max_lb_cost = jiffies,
-+#ifdef CONFIG_SCHED_DEBUG
-+ .name = tl->name,
-+#endif
-+ };
-+
-+ /*
-+ * Convert topological properties into behaviour.
-+ */
-+
-+ if (sd->flags & SD_SHARE_CPUCAPACITY) {
-+ sd->flags |= SD_PREFER_SIBLING;
-+ sd->imbalance_pct = 110;
-+ sd->smt_gain = 1178; /* ~15% */
-+
-+ } else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
-+ sd->imbalance_pct = 117;
-+ sd->cache_nice_tries = 1;
-+ sd->busy_idx = 2;
-+
-+#ifdef CONFIG_NUMA
-+ } else if (sd->flags & SD_NUMA) {
-+ sd->cache_nice_tries = 2;
-+ sd->busy_idx = 3;
-+ sd->idle_idx = 2;
-+
-+ sd->flags |= SD_SERIALIZE;
-+ if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
-+ sd->flags &= ~(SD_BALANCE_EXEC |
-+ SD_BALANCE_FORK |
-+ SD_WAKE_AFFINE);
-+ }
-+
-+#endif
-+ } else {
-+ sd->flags |= SD_PREFER_SIBLING;
-+ sd->cache_nice_tries = 1;
-+ sd->busy_idx = 2;
-+ sd->idle_idx = 1;
-+ }
-+
-+ sd->private = &tl->data;
-+
-+ return sd;
-+}
-+
-+/*
-+ * Topology list, bottom-up.
-+ */
-+static struct sched_domain_topology_level default_topology[] = {
-+#ifdef CONFIG_SCHED_SMT
-+ { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
-+#endif
-+#ifdef CONFIG_SCHED_MC
-+ { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
-+#endif
-+ { cpu_cpu_mask, SD_INIT_NAME(DIE) },
-+ { NULL, },
-+};
-+
-+struct sched_domain_topology_level *sched_domain_topology = default_topology;
-+
-+#define for_each_sd_topology(tl) \
-+ for (tl = sched_domain_topology; tl->mask; tl++)
-+
-+void set_sched_topology(struct sched_domain_topology_level *tl)
-+{
-+ sched_domain_topology = tl;
-+}
-+
-+#ifdef CONFIG_NUMA
-+
-+static const struct cpumask *sd_numa_mask(int cpu)
-+{
-+ return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
-+}
-+
-+static void sched_numa_warn(const char *str)
-+{
-+ static int done = false;
-+ int i,j;
-+
-+ if (done)
-+ return;
-+
-+ done = true;
-+
-+ printk(KERN_WARNING "ERROR: %s\n\n", str);
-+
-+ for (i = 0; i < nr_node_ids; i++) {
-+ printk(KERN_WARNING " ");
-+ for (j = 0; j < nr_node_ids; j++)
-+ printk(KERN_CONT "%02d ", node_distance(i,j));
-+ printk(KERN_CONT "\n");
-+ }
-+ printk(KERN_WARNING "\n");
-+}
-+
-+bool find_numa_distance(int distance)
-+{
-+ int i;
-+
-+ if (distance == node_distance(0, 0))
-+ return true;
-+
-+ for (i = 0; i < sched_domains_numa_levels; i++) {
-+ if (sched_domains_numa_distance[i] == distance)
-+ return true;
-+ }
-+
-+ return false;
-+}
-+
-+/*
-+ * A system can have three types of NUMA topology:
-+ * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system
-+ * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes
-+ * NUMA_BACKPLANE: nodes can reach other nodes through a backplane
-+ *
-+ * The difference between a glueless mesh topology and a backplane
-+ * topology lies in whether communication between not directly
-+ * connected nodes goes through intermediary nodes (where programs
-+ * could run), or through backplane controllers. This affects
-+ * placement of programs.
-+ *
-+ * The type of topology can be discerned with the following tests:
-+ * - If the maximum distance between any nodes is 1 hop, the system
-+ * is directly connected.
-+ * - If for two nodes A and B, located N > 1 hops away from each other,
-+ * there is an intermediary node C, which is < N hops away from both
-+ * nodes A and B, the system is a glueless mesh.
-+ */
-+static void init_numa_topology_type(void)
-+{
-+ int a, b, c, n;
-+
-+ n = sched_max_numa_distance;
-+
-+ if (n <= 1)
-+ sched_numa_topology_type = NUMA_DIRECT;
-+
-+ for_each_online_node(a) {
-+ for_each_online_node(b) {
-+ /* Find two nodes furthest removed from each other. */
-+ if (node_distance(a, b) < n)
-+ continue;
-+
-+ /* Is there an intermediary node between a and b? */
-+ for_each_online_node(c) {
-+ if (node_distance(a, c) < n &&
-+ node_distance(b, c) < n) {
-+ sched_numa_topology_type =
-+ NUMA_GLUELESS_MESH;
-+ return;
-+ }
-+ }
-+
-+ sched_numa_topology_type = NUMA_BACKPLANE;
-+ return;
-+ }
-+ }
-+}
-+
-+static void sched_init_numa(void)
-+{
-+ int next_distance, curr_distance = node_distance(0, 0);
-+ struct sched_domain_topology_level *tl;
-+ int level = 0;
-+ int i, j, k;
-+
-+ sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
-+ if (!sched_domains_numa_distance)
-+ return;
-+
-+ /*
-+ * O(nr_nodes^2) deduplicating selection sort -- in order to find the
-+ * unique distances in the node_distance() table.
-+ *
-+ * Assumes node_distance(0,j) includes all distances in
-+ * node_distance(i,j) in order to avoid cubic time.
-+ */
-+ next_distance = curr_distance;
-+ for (i = 0; i < nr_node_ids; i++) {
-+ for (j = 0; j < nr_node_ids; j++) {
-+ for (k = 0; k < nr_node_ids; k++) {
-+ int distance = node_distance(i, k);
-+
-+ if (distance > curr_distance &&
-+ (distance < next_distance ||
-+ next_distance == curr_distance))
-+ next_distance = distance;
-+
-+ /*
-+ * While not a strong assumption it would be nice to know
-+ * about cases where if node A is connected to B, B is not
-+ * equally connected to A.
-+ */
-+ if (sched_debug() && node_distance(k, i) != distance)
-+ sched_numa_warn("Node-distance not symmetric");
-+
-+ if (sched_debug() && i && !find_numa_distance(distance))
-+ sched_numa_warn("Node-0 not representative");
-+ }
-+ if (next_distance != curr_distance) {
-+ sched_domains_numa_distance[level++] = next_distance;
-+ sched_domains_numa_levels = level;
-+ curr_distance = next_distance;
-+ } else break;
-+ }
-+
-+ /*
-+ * In case of sched_debug() we verify the above assumption.
-+ */
-+ if (!sched_debug())
-+ break;
-+ }
-+
-+ if (!level)
-+ return;
-+
-+ /*
-+ * 'level' contains the number of unique distances, excluding the
-+ * identity distance node_distance(i,i).
-+ *
-+ * The sched_domains_numa_distance[] array includes the actual distance
-+ * numbers.
-+ */
-+
-+ /*
-+ * Here, we should temporarily reset sched_domains_numa_levels to 0.
-+ * If it fails to allocate memory for array sched_domains_numa_masks[][],
-+ * the array will contain less then 'level' members. This could be
-+ * dangerous when we use it to iterate array sched_domains_numa_masks[][]
-+ * in other functions.
-+ *
-+ * We reset it to 'level' at the end of this function.
-+ */
-+ sched_domains_numa_levels = 0;
-+
-+ sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
-+ if (!sched_domains_numa_masks)
-+ return;
-+
-+ /*
-+ * Now for each level, construct a mask per node which contains all
-+ * cpus of nodes that are that many hops away from us.
-+ */
-+ for (i = 0; i < level; i++) {
-+ sched_domains_numa_masks[i] =
-+ kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
-+ if (!sched_domains_numa_masks[i])
-+ return;
-+
-+ for (j = 0; j < nr_node_ids; j++) {
-+ struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
-+ if (!mask)
-+ return;
-+
-+ sched_domains_numa_masks[i][j] = mask;
-+
-+ for (k = 0; k < nr_node_ids; k++) {
-+ if (node_distance(j, k) > sched_domains_numa_distance[i])
-+ continue;
-+
-+ cpumask_or(mask, mask, cpumask_of_node(k));
-+ }
-+ }
-+ }
-+
-+ /* Compute default topology size */
-+ for (i = 0; sched_domain_topology[i].mask; i++);
-+
-+ tl = kzalloc((i + level + 1) *
-+ sizeof(struct sched_domain_topology_level), GFP_KERNEL);
-+ if (!tl)
-+ return;
-+
-+ /*
-+ * Copy the default topology bits..
-+ */
-+ for (i = 0; sched_domain_topology[i].mask; i++)
-+ tl[i] = sched_domain_topology[i];
-+
-+ /*
-+ * .. and append 'j' levels of NUMA goodness.
-+ */
-+ for (j = 0; j < level; i++, j++) {
-+ tl[i] = (struct sched_domain_topology_level){
-+ .mask = sd_numa_mask,
-+ .sd_flags = cpu_numa_flags,
-+ .flags = SDTL_OVERLAP,
-+ .numa_level = j,
-+ SD_INIT_NAME(NUMA)
-+ };
-+ }
-+
-+ sched_domain_topology = tl;
-+
-+ sched_domains_numa_levels = level;
-+ sched_max_numa_distance = sched_domains_numa_distance[level - 1];
-+
-+ init_numa_topology_type();
-+}
-+
-+static void sched_domains_numa_masks_set(int cpu)
-+{
-+ int i, j;
-+ int node = cpu_to_node(cpu);
-+
-+ for (i = 0; i < sched_domains_numa_levels; i++) {
-+ for (j = 0; j < nr_node_ids; j++) {
-+ if (node_distance(j, node) <= sched_domains_numa_distance[i])
-+ cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
-+ }
-+ }
-+}
-+
-+static void sched_domains_numa_masks_clear(int cpu)
-+{
-+ int i, j;
-+ for (i = 0; i < sched_domains_numa_levels; i++) {
-+ for (j = 0; j < nr_node_ids; j++)
-+ cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
-+ }
-+}
-+
-+/*
-+ * Update sched_domains_numa_masks[level][node] array when new cpus
-+ * are onlined.
-+ */
-+static int sched_domains_numa_masks_update(struct notifier_block *nfb,
-+ unsigned long action,
-+ void *hcpu)
-+{
-+ int cpu = (long)hcpu;
-+
-+ switch (action & ~CPU_TASKS_FROZEN) {
-+ case CPU_ONLINE:
-+ sched_domains_numa_masks_set(cpu);
-+ break;
-+
-+ case CPU_DEAD:
-+ sched_domains_numa_masks_clear(cpu);
-+ break;
-+
-+ default:
-+ return NOTIFY_DONE;
-+ }
-+
-+ return NOTIFY_OK;
-+}
-+#else
-+static inline void sched_init_numa(void)
-+{
-+}
-+
-+static int sched_domains_numa_masks_update(struct notifier_block *nfb,
-+ unsigned long action,
-+ void *hcpu)
-+{
-+ return 0;
-+}
-+#endif /* CONFIG_NUMA */
-+
-+static int __sdt_alloc(const struct cpumask *cpu_map)
-+{
-+ struct sched_domain_topology_level *tl;
-+ int j;
-+
-+ for_each_sd_topology(tl) {
-+ struct sd_data *sdd = &tl->data;
-+
-+ sdd->sd = alloc_percpu(struct sched_domain *);
-+ if (!sdd->sd)
-+ return -ENOMEM;
-+
-+ sdd->sg = alloc_percpu(struct sched_group *);
-+ if (!sdd->sg)
-+ return -ENOMEM;
-+
-+ sdd->sgc = alloc_percpu(struct sched_group_capacity *);
-+ if (!sdd->sgc)
-+ return -ENOMEM;
-+
-+ for_each_cpu(j, cpu_map) {
-+ struct sched_domain *sd;
-+ struct sched_group *sg;
-+ struct sched_group_capacity *sgc;
-+
-+ sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
-+ GFP_KERNEL, cpu_to_node(j));
-+ if (!sd)
-+ return -ENOMEM;
-+
-+ *per_cpu_ptr(sdd->sd, j) = sd;
-+
-+ sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
-+ GFP_KERNEL, cpu_to_node(j));
-+ if (!sg)
-+ return -ENOMEM;
-+
-+ sg->next = sg;
-+
-+ *per_cpu_ptr(sdd->sg, j) = sg;
-+
-+ sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
-+ GFP_KERNEL, cpu_to_node(j));
-+ if (!sgc)
-+ return -ENOMEM;
-+
-+ *per_cpu_ptr(sdd->sgc, j) = sgc;
-+ }
-+ }
-+
-+ return 0;
-+}
-+
-+static void __sdt_free(const struct cpumask *cpu_map)
-+{
-+ struct sched_domain_topology_level *tl;
-+ int j;
-+
-+ for_each_sd_topology(tl) {
-+ struct sd_data *sdd = &tl->data;
-+
-+ for_each_cpu(j, cpu_map) {
-+ struct sched_domain *sd;
-+
-+ if (sdd->sd) {
-+ sd = *per_cpu_ptr(sdd->sd, j);
-+ if (sd && (sd->flags & SD_OVERLAP))
-+ free_sched_groups(sd->groups, 0);
-+ kfree(*per_cpu_ptr(sdd->sd, j));
-+ }
-+
-+ if (sdd->sg)
-+ kfree(*per_cpu_ptr(sdd->sg, j));
-+ if (sdd->sgc)
-+ kfree(*per_cpu_ptr(sdd->sgc, j));
-+ }
-+ free_percpu(sdd->sd);
-+ sdd->sd = NULL;
-+ free_percpu(sdd->sg);
-+ sdd->sg = NULL;
-+ free_percpu(sdd->sgc);
-+ sdd->sgc = NULL;
-+ }
-+}
-+
-+struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
-+ const struct cpumask *cpu_map, struct sched_domain_attr *attr,
-+ struct sched_domain *child, int cpu)
-+{
-+ struct sched_domain *sd = sd_init(tl, cpu);
-+ if (!sd)
-+ return child;
-+
-+ cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
-+ if (child) {
-+ sd->level = child->level + 1;
-+ sched_domain_level_max = max(sched_domain_level_max, sd->level);
-+ child->parent = sd;
-+ sd->child = child;
-+
-+ if (!cpumask_subset(sched_domain_span(child),
-+ sched_domain_span(sd))) {
-+ pr_err("BUG: arch topology borken\n");
-+#ifdef CONFIG_SCHED_DEBUG
-+ pr_err(" the %s domain not a subset of the %s domain\n",
-+ child->name, sd->name);
-+#endif
-+ /* Fixup, ensure @sd has at least @child cpus. */
-+ cpumask_or(sched_domain_span(sd),
-+ sched_domain_span(sd),
-+ sched_domain_span(child));
-+ }
-+
-+ }
-+ set_domain_attribute(sd, attr);
-+
-+ return sd;
-+}
-+
-+/*
-+ * Build sched domains for a given set of cpus and attach the sched domains
-+ * to the individual cpus
-+ */
-+static int build_sched_domains(const struct cpumask *cpu_map,
-+ struct sched_domain_attr *attr)
-+{
-+ enum s_alloc alloc_state;
-+ struct sched_domain *sd;
-+ struct s_data d;
-+ int i, ret = -ENOMEM;
-+
-+ alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
-+ if (alloc_state != sa_rootdomain)
-+ goto error;
-+
-+ /* Set up domains for cpus specified by the cpu_map. */
-+ for_each_cpu(i, cpu_map) {
-+ struct sched_domain_topology_level *tl;
-+
-+ sd = NULL;
-+ for_each_sd_topology(tl) {
-+ sd = build_sched_domain(tl, cpu_map, attr, sd, i);
-+ if (tl == sched_domain_topology)
-+ *per_cpu_ptr(d.sd, i) = sd;
-+ if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
-+ sd->flags |= SD_OVERLAP;
-+ if (cpumask_equal(cpu_map, sched_domain_span(sd)))
-+ break;
-+ }
-+ }
-+
-+ /* Build the groups for the domains */
-+ for_each_cpu(i, cpu_map) {
-+ for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
-+ sd->span_weight = cpumask_weight(sched_domain_span(sd));
-+ if (sd->flags & SD_OVERLAP) {
-+ if (build_overlap_sched_groups(sd, i))
-+ goto error;
-+ } else {
-+ if (build_sched_groups(sd, i))
-+ goto error;
-+ }
-+ }
-+ }
-+
-+ /* Calculate CPU capacity for physical packages and nodes */
-+ for (i = nr_cpumask_bits-1; i >= 0; i--) {
-+ if (!cpumask_test_cpu(i, cpu_map))
-+ continue;
-+
-+ for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
-+ claim_allocations(i, sd);
-+ init_sched_groups_capacity(i, sd);
-+ }
-+ }
-+
-+ /* Attach the domains */
-+ rcu_read_lock();
-+ for_each_cpu(i, cpu_map) {
-+ sd = *per_cpu_ptr(d.sd, i);
-+ cpu_attach_domain(sd, d.rd, i);
-+ }
-+ rcu_read_unlock();
-+
-+ ret = 0;
-+error:
-+ __free_domain_allocs(&d, alloc_state, cpu_map);
-+ return ret;
-+}
-+
-+static cpumask_var_t *doms_cur; /* current sched domains */
-+static int ndoms_cur; /* number of sched domains in 'doms_cur' */
-+static struct sched_domain_attr *dattr_cur;
-+ /* attribues of custom domains in 'doms_cur' */
-+
-+/*
-+ * Special case: If a kmalloc of a doms_cur partition (array of
-+ * cpumask) fails, then fallback to a single sched domain,
-+ * as determined by the single cpumask fallback_doms.
-+ */
-+static cpumask_var_t fallback_doms;
-+
-+/*
-+ * arch_update_cpu_topology lets virtualized architectures update the
-+ * cpu core maps. It is supposed to return 1 if the topology changed
-+ * or 0 if it stayed the same.
-+ */
-+int __weak arch_update_cpu_topology(void)
-+{
-+ return 0;
-+}
-+
-+cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
-+{
-+ int i;
-+ cpumask_var_t *doms;
-+
-+ doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
-+ if (!doms)
-+ return NULL;
-+ for (i = 0; i < ndoms; i++) {
-+ if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
-+ free_sched_domains(doms, i);
-+ return NULL;
-+ }
-+ }
-+ return doms;
-+}
-+
-+void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
-+{
-+ unsigned int i;
-+ for (i = 0; i < ndoms; i++)
-+ free_cpumask_var(doms[i]);
-+ kfree(doms);
-+}
-+
-+/*
-+ * Set up scheduler domains and groups. Callers must hold the hotplug lock.
-+ * For now this just excludes isolated cpus, but could be used to
-+ * exclude other special cases in the future.
-+ */
-+static int init_sched_domains(const struct cpumask *cpu_map)
-+{
-+ int err;
-+
-+ arch_update_cpu_topology();
-+ ndoms_cur = 1;
-+ doms_cur = alloc_sched_domains(ndoms_cur);
-+ if (!doms_cur)
-+ doms_cur = &fallback_doms;
-+ cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
-+ err = build_sched_domains(doms_cur[0], NULL);
-+ register_sched_domain_sysctl();
-+
-+ return err;
-+}
-+
-+/*
-+ * Detach sched domains from a group of cpus specified in cpu_map
-+ * These cpus will now be attached to the NULL domain
-+ */
-+static void detach_destroy_domains(const struct cpumask *cpu_map)
-+{
-+ int i;
-+
-+ rcu_read_lock();
-+ for_each_cpu(i, cpu_map)
-+ cpu_attach_domain(NULL, &def_root_domain, i);
-+ rcu_read_unlock();
-+}
-+
-+/* handle null as "default" */
-+static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
-+ struct sched_domain_attr *new, int idx_new)
-+{
-+ struct sched_domain_attr tmp;
-+
-+ /* fast path */
-+ if (!new && !cur)
-+ return 1;
-+
-+ tmp = SD_ATTR_INIT;
-+ return !memcmp(cur ? (cur + idx_cur) : &tmp,
-+ new ? (new + idx_new) : &tmp,
-+ sizeof(struct sched_domain_attr));
-+}
-+
-+/*
-+ * Partition sched domains as specified by the 'ndoms_new'
-+ * cpumasks in the array doms_new[] of cpumasks. This compares
-+ * doms_new[] to the current sched domain partitioning, doms_cur[].
-+ * It destroys each deleted domain and builds each new domain.
-+ *
-+ * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
-+ * The masks don't intersect (don't overlap.) We should setup one
-+ * sched domain for each mask. CPUs not in any of the cpumasks will
-+ * not be load balanced. If the same cpumask appears both in the
-+ * current 'doms_cur' domains and in the new 'doms_new', we can leave
-+ * it as it is.
-+ *
-+ * The passed in 'doms_new' should be allocated using
-+ * alloc_sched_domains. This routine takes ownership of it and will
-+ * free_sched_domains it when done with it. If the caller failed the
-+ * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
-+ * and partition_sched_domains() will fallback to the single partition
-+ * 'fallback_doms', it also forces the domains to be rebuilt.
-+ *
-+ * If doms_new == NULL it will be replaced with cpu_online_mask.
-+ * ndoms_new == 0 is a special case for destroying existing domains,
-+ * and it will not create the default domain.
-+ *
-+ * Call with hotplug lock held
-+ */
-+void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
-+ struct sched_domain_attr *dattr_new)
-+{
-+ int i, j, n;
-+ int new_topology;
-+
-+ mutex_lock(&sched_domains_mutex);
-+
-+ /* always unregister in case we don't destroy any domains */
-+ unregister_sched_domain_sysctl();
-+
-+ /* Let architecture update cpu core mappings. */
-+ new_topology = arch_update_cpu_topology();
-+
-+ n = doms_new ? ndoms_new : 0;
-+
-+ /* Destroy deleted domains */
-+ for (i = 0; i < ndoms_cur; i++) {
-+ for (j = 0; j < n && !new_topology; j++) {
-+ if (cpumask_equal(doms_cur[i], doms_new[j])
-+ && dattrs_equal(dattr_cur, i, dattr_new, j))
-+ goto match1;
-+ }
-+ /* no match - a current sched domain not in new doms_new[] */
-+ detach_destroy_domains(doms_cur[i]);
-+match1:
-+ ;
-+ }
-+
-+ n = ndoms_cur;
-+ if (doms_new == NULL) {
-+ n = 0;
-+ doms_new = &fallback_doms;
-+ cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
-+ WARN_ON_ONCE(dattr_new);
-+ }
-+
-+ /* Build new domains */
-+ for (i = 0; i < ndoms_new; i++) {
-+ for (j = 0; j < n && !new_topology; j++) {
-+ if (cpumask_equal(doms_new[i], doms_cur[j])
-+ && dattrs_equal(dattr_new, i, dattr_cur, j))
-+ goto match2;
-+ }
-+ /* no match - add a new doms_new */
-+ build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
-+match2:
-+ ;
-+ }
-+
-+ /* Remember the new sched domains */
-+ if (doms_cur != &fallback_doms)
-+ free_sched_domains(doms_cur, ndoms_cur);
-+ kfree(dattr_cur); /* kfree(NULL) is safe */
-+ doms_cur = doms_new;
-+ dattr_cur = dattr_new;
-+ ndoms_cur = ndoms_new;
-+
-+ register_sched_domain_sysctl();
-+
-+ mutex_unlock(&sched_domains_mutex);
-+}
-+
-+static int num_cpus_frozen; /* used to mark begin/end of suspend/resume */
-+
-+/*
-+ * Update cpusets according to cpu_active mask. If cpusets are
-+ * disabled, cpuset_update_active_cpus() becomes a simple wrapper
-+ * around partition_sched_domains().
-+ *
-+ * If we come here as part of a suspend/resume, don't touch cpusets because we
-+ * want to restore it back to its original state upon resume anyway.
-+ */
-+static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
-+ void *hcpu)
-+{
-+ switch (action) {
-+ case CPU_ONLINE_FROZEN:
-+ case CPU_DOWN_FAILED_FROZEN:
-+
-+ /*
-+ * num_cpus_frozen tracks how many CPUs are involved in suspend
-+ * resume sequence. As long as this is not the last online
-+ * operation in the resume sequence, just build a single sched
-+ * domain, ignoring cpusets.
-+ */
-+ num_cpus_frozen--;
-+ if (likely(num_cpus_frozen)) {
-+ partition_sched_domains(1, NULL, NULL);
-+ break;
-+ }
-+
-+ /*
-+ * This is the last CPU online operation. So fall through and
-+ * restore the original sched domains by considering the
-+ * cpuset configurations.
-+ */
-+
-+ case CPU_ONLINE:
-+ cpuset_update_active_cpus(true);
-+ break;
-+ default:
-+ return NOTIFY_DONE;
-+ }
-+ return NOTIFY_OK;
-+}
-+
-+static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
-+ void *hcpu)
-+{
-+ unsigned long flags;
-+ long cpu = (long)hcpu;
-+ struct dl_bw *dl_b;
-+ bool overflow;
-+ int cpus;
-+
-+ switch (action) {
-+ case CPU_DOWN_PREPARE:
-+ rcu_read_lock_sched();
-+ dl_b = dl_bw_of(cpu);
-+
-+ raw_spin_lock_irqsave(&dl_b->lock, flags);
-+ cpus = dl_bw_cpus(cpu);
-+ overflow = __dl_overflow(dl_b, cpus, 0, 0);
-+ raw_spin_unlock_irqrestore(&dl_b->lock, flags);
-+
-+ rcu_read_unlock_sched();
-+
-+ if (overflow)
-+ return notifier_from_errno(-EBUSY);
-+ cpuset_update_active_cpus(false);
-+ break;
-+ case CPU_DOWN_PREPARE_FROZEN:
-+ num_cpus_frozen++;
-+ partition_sched_domains(1, NULL, NULL);
-+ break;
-+ default:
-+ return NOTIFY_DONE;
-+ }
-+ return NOTIFY_OK;
-+}
-+
-+void __init sched_init_smp(void)
-+{
-+ cpumask_var_t non_isolated_cpus;
-+
-+ alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
-+ alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
-+
-+ sched_init_numa();
-+
-+ /*
-+ * There's no userspace yet to cause hotplug operations; hence all the
-+ * cpu masks are stable and all blatant races in the below code cannot
-+ * happen.
-+ */
-+ mutex_lock(&sched_domains_mutex);
-+ init_sched_domains(cpu_active_mask);
-+ cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
-+ if (cpumask_empty(non_isolated_cpus))
-+ cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
-+ mutex_unlock(&sched_domains_mutex);
-+
-+ hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
-+ hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
-+ hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
-+
-+ init_hrtick();
-+
-+ /* Move init over to a non-isolated CPU */
-+ if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
-+ BUG();
-+ sched_init_granularity();
-+ free_cpumask_var(non_isolated_cpus);
-+
-+ init_sched_rt_class();
-+ init_sched_dl_class();
-+}
-+#else
-+void __init sched_init_smp(void)
-+{
-+ sched_init_granularity();
-+}
-+#endif /* CONFIG_SMP */
-+
-+const_debug unsigned int sysctl_timer_migration = 1;
-+
-+int in_sched_functions(unsigned long addr)
-+{
-+ return in_lock_functions(addr) ||
-+ (addr >= (unsigned long)__sched_text_start
-+ && addr < (unsigned long)__sched_text_end);
-+}
-+
-+#ifdef CONFIG_CGROUP_SCHED
-+/*
-+ * Default task group.
-+ * Every task in system belongs to this group at bootup.
-+ */
-+struct task_group root_task_group;
-+LIST_HEAD(task_groups);
-+#endif
-+
-+DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
-+
-+void __init sched_init(void)
-+{
-+ int i, j;
-+ unsigned long alloc_size = 0, ptr;
-+
-+#ifdef CONFIG_FAIR_GROUP_SCHED
-+ alloc_size += 2 * nr_cpu_ids * sizeof(void **);
-+#endif
-+#ifdef CONFIG_RT_GROUP_SCHED
-+ alloc_size += 2 * nr_cpu_ids * sizeof(void **);
-+#endif
-+ if (alloc_size) {
-+ ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
-+
-+#ifdef CONFIG_FAIR_GROUP_SCHED
-+ root_task_group.se = (struct sched_entity **)ptr;
-+ ptr += nr_cpu_ids * sizeof(void **);
-+
-+ root_task_group.cfs_rq = (struct cfs_rq **)ptr;
-+ ptr += nr_cpu_ids * sizeof(void **);
-+
-+#endif /* CONFIG_FAIR_GROUP_SCHED */
-+#ifdef CONFIG_RT_GROUP_SCHED
-+ root_task_group.rt_se = (struct sched_rt_entity **)ptr;
-+ ptr += nr_cpu_ids * sizeof(void **);
-+
-+ root_task_group.rt_rq = (struct rt_rq **)ptr;
-+ ptr += nr_cpu_ids * sizeof(void **);
-+
-+#endif /* CONFIG_RT_GROUP_SCHED */
-+ }
-+#ifdef CONFIG_CPUMASK_OFFSTACK
-+ for_each_possible_cpu(i) {
-+ per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node(
-+ cpumask_size(), GFP_KERNEL, cpu_to_node(i));
-+ }
-+#endif /* CONFIG_CPUMASK_OFFSTACK */
-+
-+ init_rt_bandwidth(&def_rt_bandwidth,
-+ global_rt_period(), global_rt_runtime());
-+ init_dl_bandwidth(&def_dl_bandwidth,
-+ global_rt_period(), global_rt_runtime());
-+
-+#ifdef CONFIG_SMP
-+ init_defrootdomain();
-+#endif
-+
-+#ifdef CONFIG_RT_GROUP_SCHED
-+ init_rt_bandwidth(&root_task_group.rt_bandwidth,
-+ global_rt_period(), global_rt_runtime());
-+#endif /* CONFIG_RT_GROUP_SCHED */
-+
-+#ifdef CONFIG_CGROUP_SCHED
-+ list_add(&root_task_group.list, &task_groups);
-+ INIT_LIST_HEAD(&root_task_group.children);
-+ INIT_LIST_HEAD(&root_task_group.siblings);
-+ autogroup_init(&init_task);
-+
-+#endif /* CONFIG_CGROUP_SCHED */
-+
-+ for_each_possible_cpu(i) {
-+ struct rq *rq;
-+
-+ rq = cpu_rq(i);
-+ raw_spin_lock_init(&rq->lock);
-+ rq->nr_running = 0;
-+ rq->calc_load_active = 0;
-+ rq->calc_load_update = jiffies + LOAD_FREQ;
-+ init_cfs_rq(&rq->cfs);
-+ init_rt_rq(&rq->rt);
-+ init_dl_rq(&rq->dl);
-+#ifdef CONFIG_FAIR_GROUP_SCHED
-+ root_task_group.shares = ROOT_TASK_GROUP_LOAD;
-+ INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
-+ /*
-+ * How much cpu bandwidth does root_task_group get?
-+ *
-+ * In case of task-groups formed thr' the cgroup filesystem, it
-+ * gets 100% of the cpu resources in the system. This overall
-+ * system cpu resource is divided among the tasks of
-+ * root_task_group and its child task-groups in a fair manner,
-+ * based on each entity's (task or task-group's) weight
-+ * (se->load.weight).
-+ *
-+ * In other words, if root_task_group has 10 tasks of weight
-+ * 1024) and two child groups A0 and A1 (of weight 1024 each),
-+ * then A0's share of the cpu resource is:
-+ *
-+ * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
-+ *
-+ * We achieve this by letting root_task_group's tasks sit
-+ * directly in rq->cfs (i.e root_task_group->se[] = NULL).
-+ */
-+ init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
-+ init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
-+#endif /* CONFIG_FAIR_GROUP_SCHED */
-+
-+ rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
-+#ifdef CONFIG_RT_GROUP_SCHED
-+ init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
-+#endif
-+
-+ for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
-+ rq->cpu_load[j] = 0;
-+
-+ rq->last_load_update_tick = jiffies;
-+
-+#ifdef CONFIG_SMP
-+ rq->sd = NULL;
-+ rq->rd = NULL;
-+ rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE;
-+ rq->post_schedule = 0;
-+ rq->active_balance = 0;
-+ rq->next_balance = jiffies;
-+ rq->push_cpu = 0;
-+ rq->cpu = i;
-+ rq->online = 0;
-+ rq->idle_stamp = 0;
-+ rq->avg_idle = 2*sysctl_sched_migration_cost;
-+ rq->max_idle_balance_cost = sysctl_sched_migration_cost;
-+
-+ INIT_LIST_HEAD(&rq->cfs_tasks);
-+
-+ rq_attach_root(rq, &def_root_domain);
-+#ifdef CONFIG_NO_HZ_COMMON
-+ rq->nohz_flags = 0;
-+#endif
-+#ifdef CONFIG_NO_HZ_FULL
-+ rq->last_sched_tick = 0;
-+#endif
-+#endif
-+ init_rq_hrtick(rq);
-+ atomic_set(&rq->nr_iowait, 0);
-+ }
-+
-+ set_load_weight(&init_task);
-+
-+#ifdef CONFIG_PREEMPT_NOTIFIERS
-+ INIT_HLIST_HEAD(&init_task.preempt_notifiers);
-+#endif
-+
-+ /*
-+ * The boot idle thread does lazy MMU switching as well:
-+ */
-+ atomic_inc(&init_mm.mm_count);
-+ enter_lazy_tlb(&init_mm, current);
-+
-+ /*
-+ * During early bootup we pretend to be a normal task:
-+ */
-+ current->sched_class = &fair_sched_class;
-+
-+ /*
-+ * Make us the idle thread. Technically, schedule() should not be
-+ * called from this thread, however somewhere below it might be,
-+ * but because we are the idle thread, we just pick up running again
-+ * when this runqueue becomes "idle".
-+ */
-+ init_idle(current, smp_processor_id());
-+
-+ calc_load_update = jiffies + LOAD_FREQ;
-+
-+#ifdef CONFIG_SMP
-+ zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
-+ /* May be allocated at isolcpus cmdline parse time */
-+ if (cpu_isolated_map == NULL)
-+ zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
-+ idle_thread_set_boot_cpu();
-+ set_cpu_rq_start_time();
-+#endif
-+ init_sched_fair_class();
-+
-+ scheduler_running = 1;
-+}
-+
-+#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
-+static inline int preempt_count_equals(int preempt_offset)
-+{
-+ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
-+
-+ return (nested == preempt_offset);
-+}
-+
-+void __might_sleep(const char *file, int line, int preempt_offset)
-+{
-+ /*
-+ * Blocking primitives will set (and therefore destroy) current->state,
-+ * since we will exit with TASK_RUNNING make sure we enter with it,
-+ * otherwise we will destroy state.
-+ */
-+ WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
-+ "do not call blocking ops when !TASK_RUNNING; "
-+ "state=%lx set at [<%p>] %pS\n",
-+ current->state,
-+ (void *)current->task_state_change,
-+ (void *)current->task_state_change);
-+
-+ ___might_sleep(file, line, preempt_offset);
-+}
-+EXPORT_SYMBOL(__might_sleep);
-+
-+void ___might_sleep(const char *file, int line, int preempt_offset)
-+{
-+ static unsigned long prev_jiffy; /* ratelimiting */
-+
-+ rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
-+ if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
-+ !is_idle_task(current)) ||
-+ system_state != SYSTEM_RUNNING || oops_in_progress)
-+ return;
-+ if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
-+ return;
-+ prev_jiffy = jiffies;
-+
-+ printk(KERN_ERR
-+ "BUG: sleeping function called from invalid context at %s:%d\n",
-+ file, line);
-+ printk(KERN_ERR
-+ "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
-+ in_atomic(), irqs_disabled(),
-+ current->pid, current->comm);
-+
-+ if (task_stack_end_corrupted(current))
-+ printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
-+
-+ debug_show_held_locks(current);
-+ if (irqs_disabled())
-+ print_irqtrace_events(current);
-+#ifdef CONFIG_DEBUG_PREEMPT
-+ if (!preempt_count_equals(preempt_offset)) {
-+ pr_err("Preemption disabled at:");
-+ print_ip_sym(current->preempt_disable_ip);
-+ pr_cont("\n");
-+ }
-+#endif
-+ dump_stack();
-+}
-+EXPORT_SYMBOL(___might_sleep);
-+#endif
-+
-+#ifdef CONFIG_MAGIC_SYSRQ
-+static void normalize_task(struct rq *rq, struct task_struct *p)
-+{
-+ const struct sched_class *prev_class = p->sched_class;
-+ struct sched_attr attr = {
-+ .sched_policy = SCHED_NORMAL,
-+ };
-+ int old_prio = p->prio;
-+ int queued;
-+
-+ queued = task_on_rq_queued(p);
-+ if (queued)
-+ dequeue_task(rq, p, 0);
-+ __setscheduler(rq, p, &attr, false);
-+ if (queued) {
-+ enqueue_task(rq, p, 0);
-+ resched_curr(rq);
-+ }
-+
-+ check_class_changed(rq, p, prev_class, old_prio);
-+}
-+
-+void normalize_rt_tasks(void)
-+{
-+ struct task_struct *g, *p;
-+ unsigned long flags;
-+ struct rq *rq;
-+
-+ read_lock(&tasklist_lock);
-+ for_each_process_thread(g, p) {
-+ /*
-+ * Only normalize user tasks:
-+ */
-+ if (p->flags & PF_KTHREAD)
-+ continue;
-+
-+ p->se.exec_start = 0;
-+#ifdef CONFIG_SCHEDSTATS
-+ p->se.statistics.wait_start = 0;
-+ p->se.statistics.sleep_start = 0;
-+ p->se.statistics.block_start = 0;
-+#endif
-+
-+ if (!dl_task(p) && !rt_task(p)) {
-+ /*
-+ * Renice negative nice level userspace
-+ * tasks back to 0:
-+ */
-+ if (task_nice(p) < 0)
-+ set_user_nice(p, 0);
-+ continue;
-+ }
-+
-+ rq = task_rq_lock(p, &flags);
-+ normalize_task(rq, p);
-+ task_rq_unlock(rq, p, &flags);
-+ }
-+ read_unlock(&tasklist_lock);
-+}
-+
-+#endif /* CONFIG_MAGIC_SYSRQ */
-+
-+#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
-+/*
-+ * These functions are only useful for the IA64 MCA handling, or kdb.
-+ *
-+ * They can only be called when the whole system has been
-+ * stopped - every CPU needs to be quiescent, and no scheduling
-+ * activity can take place. Using them for anything else would
-+ * be a serious bug, and as a result, they aren't even visible
-+ * under any other configuration.
-+ */
-+
-+/**
-+ * curr_task - return the current task for a given cpu.
-+ * @cpu: the processor in question.
-+ *
-+ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
-+ *
-+ * Return: The current task for @cpu.
-+ */
-+struct task_struct *curr_task(int cpu)
-+{
-+ return cpu_curr(cpu);
-+}
-+
-+#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
-+
-+#ifdef CONFIG_IA64
-+/**
-+ * set_curr_task - set the current task for a given cpu.
-+ * @cpu: the processor in question.
-+ * @p: the task pointer to set.
-+ *
-+ * Description: This function must only be used when non-maskable interrupts
-+ * are serviced on a separate stack. It allows the architecture to switch the
-+ * notion of the current task on a cpu in a non-blocking manner. This function
-+ * must be called with all CPU's synchronized, and interrupts disabled, the
-+ * and caller must save the original value of the current task (see
-+ * curr_task() above) and restore that value before reenabling interrupts and
-+ * re-starting the system.
-+ *
-+ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
-+ */
-+void set_curr_task(int cpu, struct task_struct *p)
-+{
-+ cpu_curr(cpu) = p;
-+}
-+
-+#endif
-+
-+#ifdef CONFIG_CGROUP_SCHED
-+/* task_group_lock serializes the addition/removal of task groups */
-+static DEFINE_SPINLOCK(task_group_lock);
-+
-+static void free_sched_group(struct task_group *tg)
-+{
-+ free_fair_sched_group(tg);
-+ free_rt_sched_group(tg);
-+ autogroup_free(tg);
-+ kfree(tg);
-+}
-+
-+/* allocate runqueue etc for a new task group */
-+struct task_group *sched_create_group(struct task_group *parent)
-+{
-+ struct task_group *tg;
-+
-+ tg = kzalloc(sizeof(*tg), GFP_KERNEL);
-+ if (!tg)
-+ return ERR_PTR(-ENOMEM);
-+
-+ if (!alloc_fair_sched_group(tg, parent))
-+ goto err;
-+
-+ if (!alloc_rt_sched_group(tg, parent))
-+ goto err;
-+
-+ return tg;
-+
-+err:
-+ free_sched_group(tg);
-+ return ERR_PTR(-ENOMEM);
-+}
-+
-+void sched_online_group(struct task_group *tg, struct task_group *parent)
-+{
-+ unsigned long flags;
-+
-+ spin_lock_irqsave(&task_group_lock, flags);
-+ list_add_rcu(&tg->list, &task_groups);
-+
-+ WARN_ON(!parent); /* root should already exist */
-+
-+ tg->parent = parent;
-+ INIT_LIST_HEAD(&tg->children);
-+ list_add_rcu(&tg->siblings, &parent->children);
-+ spin_unlock_irqrestore(&task_group_lock, flags);
-+}
-+
-+/* rcu callback to free various structures associated with a task group */
-+static void free_sched_group_rcu(struct rcu_head *rhp)
-+{
-+ /* now it should be safe to free those cfs_rqs */
-+ free_sched_group(container_of(rhp, struct task_group, rcu));
-+}
-+
-+/* Destroy runqueue etc associated with a task group */
-+void sched_destroy_group(struct task_group *tg)
-+{
-+ /* wait for possible concurrent references to cfs_rqs complete */
-+ call_rcu(&tg->rcu, free_sched_group_rcu);
-+}
-+
-+void sched_offline_group(struct task_group *tg)
-+{
-+ unsigned long flags;
-+ int i;
-+
-+ /* end participation in shares distribution */
-+ for_each_possible_cpu(i)
-+ unregister_fair_sched_group(tg, i);
-+
-+ spin_lock_irqsave(&task_group_lock, flags);
-+ list_del_rcu(&tg->list);
-+ list_del_rcu(&tg->siblings);
-+ spin_unlock_irqrestore(&task_group_lock, flags);
-+}
-+
-+/* change task's runqueue when it moves between groups.
-+ * The caller of this function should have put the task in its new group
-+ * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
-+ * reflect its new group.
-+ */
-+void sched_move_task(struct task_struct *tsk)
-+{
-+ struct task_group *tg;
-+ int queued, running;
-+ unsigned long flags;
-+ struct rq *rq;
-+
-+ rq = task_rq_lock(tsk, &flags);
-+
-+ running = task_current(rq, tsk);
-+ queued = task_on_rq_queued(tsk);
-+
-+ if (queued)
-+ dequeue_task(rq, tsk, 0);
-+ if (unlikely(running))
-+ put_prev_task(rq, tsk);
-+
-+ /*
-+ * All callers are synchronized by task_rq_lock(); we do not use RCU
-+ * which is pointless here. Thus, we pass "true" to task_css_check()
-+ * to prevent lockdep warnings.
-+ */
-+ tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
-+ struct task_group, css);
-+ tg = autogroup_task_group(tsk, tg);
-+ tsk->sched_task_group = tg;
-+
-+#ifdef CONFIG_FAIR_GROUP_SCHED
-+ if (tsk->sched_class->task_move_group)
-+ tsk->sched_class->task_move_group(tsk, queued);
-+ else
-+#endif
-+ set_task_rq(tsk, task_cpu(tsk));
-+
-+ if (unlikely(running))
-+ tsk->sched_class->set_curr_task(rq);
-+ if (queued)
-+ enqueue_task(rq, tsk, 0);
-+
-+ task_rq_unlock(rq, tsk, &flags);
-+}
-+#endif /* CONFIG_CGROUP_SCHED */
-+
-+#ifdef CONFIG_RT_GROUP_SCHED
-+/*
-+ * Ensure that the real time constraints are schedulable.
-+ */
-+static DEFINE_MUTEX(rt_constraints_mutex);
-+
-+/* Must be called with tasklist_lock held */
-+static inline int tg_has_rt_tasks(struct task_group *tg)
-+{
-+ struct task_struct *g, *p;
-+
-+ /*
-+ * Autogroups do not have RT tasks; see autogroup_create().
-+ */
-+ if (task_group_is_autogroup(tg))
-+ return 0;
-+
-+ for_each_process_thread(g, p) {
-+ if (rt_task(p) && task_group(p) == tg)
-+ return 1;
-+ }
-+
-+ return 0;
-+}
-+
-+struct rt_schedulable_data {
-+ struct task_group *tg;
-+ u64 rt_period;
-+ u64 rt_runtime;
-+};
-+
-+static int tg_rt_schedulable(struct task_group *tg, void *data)
-+{
-+ struct rt_schedulable_data *d = data;
-+ struct task_group *child;
-+ unsigned long total, sum = 0;
-+ u64 period, runtime;
-+
-+ period = ktime_to_ns(tg->rt_bandwidth.rt_period);
-+ runtime = tg->rt_bandwidth.rt_runtime;
-+
-+ if (tg == d->tg) {
-+ period = d->rt_period;
-+ runtime = d->rt_runtime;
-+ }
-+
-+ /*
-+ * Cannot have more runtime than the period.
-+ */
-+ if (runtime > period && runtime != RUNTIME_INF)
-+ return -EINVAL;
-+
-+ /*
-+ * Ensure we don't starve existing RT tasks.
-+ */
-+ if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
-+ return -EBUSY;
-+
-+ total = to_ratio(period, runtime);
-+
-+ /*
-+ * Nobody can have more than the global setting allows.
-+ */
-+ if (total > to_ratio(global_rt_period(), global_rt_runtime()))
-+ return -EINVAL;
-+
-+ /*
-+ * The sum of our children's runtime should not exceed our own.
-+ */
-+ list_for_each_entry_rcu(child, &tg->children, siblings) {
-+ period = ktime_to_ns(child->rt_bandwidth.rt_period);
-+ runtime = child->rt_bandwidth.rt_runtime;
-+
-+ if (child == d->tg) {
-+ period = d->rt_period;
-+ runtime = d->rt_runtime;
-+ }
-+
-+ sum += to_ratio(period, runtime);
-+ }
-+
-+ if (sum > total)
-+ return -EINVAL;
-+
-+ return 0;
-+}
-+
-+static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
-+{
-+ int ret;
-+
-+ struct rt_schedulable_data data = {
-+ .tg = tg,
-+ .rt_period = period,
-+ .rt_runtime = runtime,
-+ };
-+
-+ rcu_read_lock();
-+ ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
-+ rcu_read_unlock();
-+
-+ return ret;
-+}
-+
-+static int tg_set_rt_bandwidth(struct task_group *tg,
-+ u64 rt_period, u64 rt_runtime)
-+{
-+ int i, err = 0;
-+
-+ /*
-+ * Disallowing the root group RT runtime is BAD, it would disallow the
-+ * kernel creating (and or operating) RT threads.
-+ */
-+ if (tg == &root_task_group && rt_runtime == 0)
-+ return -EINVAL;
-+
-+ /* No period doesn't make any sense. */
-+ if (rt_period == 0)
-+ return -EINVAL;
-+
-+ mutex_lock(&rt_constraints_mutex);
-+ read_lock(&tasklist_lock);
-+ err = __rt_schedulable(tg, rt_period, rt_runtime);
-+ if (err)
-+ goto unlock;
-+
-+ raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
-+ tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
-+ tg->rt_bandwidth.rt_runtime = rt_runtime;
-+
-+ for_each_possible_cpu(i) {
-+ struct rt_rq *rt_rq = tg->rt_rq[i];
-+
-+ raw_spin_lock(&rt_rq->rt_runtime_lock);
-+ rt_rq->rt_runtime = rt_runtime;
-+ raw_spin_unlock(&rt_rq->rt_runtime_lock);
-+ }
-+ raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
-+unlock:
-+ read_unlock(&tasklist_lock);
-+ mutex_unlock(&rt_constraints_mutex);
-+
-+ return err;
-+}
-+
-+static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
-+{
-+ u64 rt_runtime, rt_period;
-+
-+ rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
-+ rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
-+ if (rt_runtime_us < 0)
-+ rt_runtime = RUNTIME_INF;
-+
-+ return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
-+}
-+
-+static long sched_group_rt_runtime(struct task_group *tg)
-+{
-+ u64 rt_runtime_us;
-+
-+ if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
-+ return -1;
-+
-+ rt_runtime_us = tg->rt_bandwidth.rt_runtime;
-+ do_div(rt_runtime_us, NSEC_PER_USEC);
-+ return rt_runtime_us;
-+}
-+
-+static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
-+{
-+ u64 rt_runtime, rt_period;
-+
-+ rt_period = (u64)rt_period_us * NSEC_PER_USEC;
-+ rt_runtime = tg->rt_bandwidth.rt_runtime;
-+
-+ return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
-+}
-+
-+static long sched_group_rt_period(struct task_group *tg)
-+{
-+ u64 rt_period_us;
-+
-+ rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
-+ do_div(rt_period_us, NSEC_PER_USEC);
-+ return rt_period_us;
-+}
-+#endif /* CONFIG_RT_GROUP_SCHED */
-+
-+#ifdef CONFIG_RT_GROUP_SCHED
-+static int sched_rt_global_constraints(void)
-+{
-+ int ret = 0;
-+
-+ mutex_lock(&rt_constraints_mutex);
-+ read_lock(&tasklist_lock);
-+ ret = __rt_schedulable(NULL, 0, 0);
-+ read_unlock(&tasklist_lock);
-+ mutex_unlock(&rt_constraints_mutex);
-+
-+ return ret;
-+}
-+
-+static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
-+{
-+ /* Don't accept realtime tasks when there is no way for them to run */
-+ if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
-+ return 0;
-+
-+ return 1;
-+}
-+
-+#else /* !CONFIG_RT_GROUP_SCHED */
-+static int sched_rt_global_constraints(void)
-+{
-+ unsigned long flags;
-+ int i, ret = 0;
-+
-+ raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
-+ for_each_possible_cpu(i) {
-+ struct rt_rq *rt_rq = &cpu_rq(i)->rt;
-+
-+ raw_spin_lock(&rt_rq->rt_runtime_lock);
-+ rt_rq->rt_runtime = global_rt_runtime();
-+ raw_spin_unlock(&rt_rq->rt_runtime_lock);
-+ }
-+ raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
-+
-+ return ret;
-+}
-+#endif /* CONFIG_RT_GROUP_SCHED */
-+
-+static int sched_dl_global_validate(void)
-+{
-+ u64 runtime = global_rt_runtime();
-+ u64 period = global_rt_period();
-+ u64 new_bw = to_ratio(period, runtime);
-+ struct dl_bw *dl_b;
-+ int cpu, ret = 0;
-+ unsigned long flags;
-+
-+ /*
-+ * Here we want to check the bandwidth not being set to some
-+ * value smaller than the currently allocated bandwidth in
-+ * any of the root_domains.
-+ *
-+ * FIXME: Cycling on all the CPUs is overdoing, but simpler than
-+ * cycling on root_domains... Discussion on different/better
-+ * solutions is welcome!
-+ */
-+ for_each_possible_cpu(cpu) {
-+ rcu_read_lock_sched();
-+ dl_b = dl_bw_of(cpu);
-+
-+ raw_spin_lock_irqsave(&dl_b->lock, flags);
-+ if (new_bw < dl_b->total_bw)
-+ ret = -EBUSY;
-+ raw_spin_unlock_irqrestore(&dl_b->lock, flags);
-+
-+ rcu_read_unlock_sched();
-+
-+ if (ret)
-+ break;
-+ }
-+
-+ return ret;
-+}
-+
-+static void sched_dl_do_global(void)
-+{
-+ u64 new_bw = -1;
-+ struct dl_bw *dl_b;
-+ int cpu;
-+ unsigned long flags;
-+
-+ def_dl_bandwidth.dl_period = global_rt_period();
-+ def_dl_bandwidth.dl_runtime = global_rt_runtime();
-+
-+ if (global_rt_runtime() != RUNTIME_INF)
-+ new_bw = to_ratio(global_rt_period(), global_rt_runtime());
-+
-+ /*
-+ * FIXME: As above...
-+ */
-+ for_each_possible_cpu(cpu) {
-+ rcu_read_lock_sched();
-+ dl_b = dl_bw_of(cpu);
-+
-+ raw_spin_lock_irqsave(&dl_b->lock, flags);
-+ dl_b->bw = new_bw;
-+ raw_spin_unlock_irqrestore(&dl_b->lock, flags);
-+
-+ rcu_read_unlock_sched();
-+ }
-+}
-+
-+static int sched_rt_global_validate(void)
-+{
-+ if (sysctl_sched_rt_period <= 0)
-+ return -EINVAL;
-+
-+ if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
-+ (sysctl_sched_rt_runtime > sysctl_sched_rt_period))
-+ return -EINVAL;
-+
-+ return 0;
-+}
-+
-+static void sched_rt_do_global(void)
-+{
-+ def_rt_bandwidth.rt_runtime = global_rt_runtime();
-+ def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
-+}
-+
-+int sched_rt_handler(struct ctl_table *table, int write,
-+ void __user *buffer, size_t *lenp,
-+ loff_t *ppos)
-+{
-+ int old_period, old_runtime;
-+ static DEFINE_MUTEX(mutex);
-+ int ret;
-+
-+ mutex_lock(&mutex);
-+ old_period = sysctl_sched_rt_period;
-+ old_runtime = sysctl_sched_rt_runtime;
-+
-+ ret = proc_dointvec(table, write, buffer, lenp, ppos);
-+
-+ if (!ret && write) {
-+ ret = sched_rt_global_validate();
-+ if (ret)
-+ goto undo;
-+
-+ ret = sched_dl_global_validate();
-+ if (ret)
-+ goto undo;
-+
-+ ret = sched_rt_global_constraints();
-+ if (ret)
-+ goto undo;
-+
-+ sched_rt_do_global();
-+ sched_dl_do_global();
-+ }
-+ if (0) {
-+undo:
-+ sysctl_sched_rt_period = old_period;
-+ sysctl_sched_rt_runtime = old_runtime;
-+ }
-+ mutex_unlock(&mutex);
-+
-+ return ret;
-+}
-+
-+int sched_rr_handler(struct ctl_table *table, int write,
-+ void __user *buffer, size_t *lenp,
-+ loff_t *ppos)
-+{
-+ int ret;
-+ static DEFINE_MUTEX(mutex);
-+
-+ mutex_lock(&mutex);
-+ ret = proc_dointvec(table, write, buffer, lenp, ppos);
-+ /* make sure that internally we keep jiffies */
-+ /* also, writing zero resets timeslice to default */
-+ if (!ret && write) {
-+ sched_rr_timeslice = sched_rr_timeslice <= 0 ?
-+ RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice);
-+ }
-+ mutex_unlock(&mutex);
-+ return ret;
-+}
-+
-+#ifdef CONFIG_CGROUP_SCHED
-+
-+static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
-+{
-+ return css ? container_of(css, struct task_group, css) : NULL;
-+}
-+
-+static struct cgroup_subsys_state *
-+cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
-+{
-+ struct task_group *parent = css_tg(parent_css);
-+ struct task_group *tg;
-+
-+ if (!parent) {
-+ /* This is early initialization for the top cgroup */
-+ return &root_task_group.css;
-+ }
-+
-+ tg = sched_create_group(parent);
-+ if (IS_ERR(tg))
-+ return ERR_PTR(-ENOMEM);
-+
-+ return &tg->css;
-+}
-+
-+static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
-+{
-+ struct task_group *tg = css_tg(css);
-+ struct task_group *parent = css_tg(css->parent);
-+
-+ if (parent)
-+ sched_online_group(tg, parent);
-+ return 0;
-+}
-+
-+static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
-+{
-+ struct task_group *tg = css_tg(css);
-+
-+ sched_destroy_group(tg);
-+}
-+
-+static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
-+{
-+ struct task_group *tg = css_tg(css);
-+
-+ sched_offline_group(tg);
-+}
-+
-+static void cpu_cgroup_fork(struct task_struct *task)
-+{
-+ sched_move_task(task);
-+}
-+
-+static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css,
-+ struct cgroup_taskset *tset)
-+{
-+ struct task_struct *task;
-+
-+ cgroup_taskset_for_each(task, tset) {
-+#ifdef CONFIG_RT_GROUP_SCHED
-+ if (!sched_rt_can_attach(css_tg(css), task))
-+ return -EINVAL;
-+#else
-+ /* We don't support RT-tasks being in separate groups */
-+ if (task->sched_class != &fair_sched_class)
-+ return -EINVAL;
-+#endif
-+ }
-+ return 0;
-+}
-+
-+static void cpu_cgroup_attach(struct cgroup_subsys_state *css,
-+ struct cgroup_taskset *tset)
-+{
-+ struct task_struct *task;
-+
-+ cgroup_taskset_for_each(task, tset)
-+ sched_move_task(task);
-+}
-+
-+static void cpu_cgroup_exit(struct cgroup_subsys_state *css,
-+ struct cgroup_subsys_state *old_css,
-+ struct task_struct *task)
-+{
-+ /*
-+ * cgroup_exit() is called in the copy_process() failure path.
-+ * Ignore this case since the task hasn't ran yet, this avoids
-+ * trying to poke a half freed task state from generic code.
-+ */
-+ if (!(task->flags & PF_EXITING))
-+ return;
-+
-+ sched_move_task(task);
-+}
-+
-+#ifdef CONFIG_FAIR_GROUP_SCHED
-+static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
-+ struct cftype *cftype, u64 shareval)
-+{
-+ return sched_group_set_shares(css_tg(css), scale_load(shareval));
-+}
-+
-+static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ struct task_group *tg = css_tg(css);
-+
-+ return (u64) scale_load_down(tg->shares);
-+}
-+
-+#ifdef CONFIG_CFS_BANDWIDTH
-+static DEFINE_MUTEX(cfs_constraints_mutex);
-+
-+const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
-+const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
-+
-+static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
-+
-+static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
-+{
-+ int i, ret = 0, runtime_enabled, runtime_was_enabled;
-+ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
-+
-+ if (tg == &root_task_group)
-+ return -EINVAL;
-+
-+ /*
-+ * Ensure we have at some amount of bandwidth every period. This is
-+ * to prevent reaching a state of large arrears when throttled via
-+ * entity_tick() resulting in prolonged exit starvation.
-+ */
-+ if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
-+ return -EINVAL;
-+
-+ /*
-+ * Likewise, bound things on the otherside by preventing insane quota
-+ * periods. This also allows us to normalize in computing quota
-+ * feasibility.
-+ */
-+ if (period > max_cfs_quota_period)
-+ return -EINVAL;
-+
-+ /*
-+ * Prevent race between setting of cfs_rq->runtime_enabled and
-+ * unthrottle_offline_cfs_rqs().
-+ */
-+ get_online_cpus();
-+ mutex_lock(&cfs_constraints_mutex);
-+ ret = __cfs_schedulable(tg, period, quota);
-+ if (ret)
-+ goto out_unlock;
-+
-+ runtime_enabled = quota != RUNTIME_INF;
-+ runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
-+ /*
-+ * If we need to toggle cfs_bandwidth_used, off->on must occur
-+ * before making related changes, and on->off must occur afterwards
-+ */
-+ if (runtime_enabled && !runtime_was_enabled)
-+ cfs_bandwidth_usage_inc();
-+ raw_spin_lock_irq(&cfs_b->lock);
-+ cfs_b->period = ns_to_ktime(period);
-+ cfs_b->quota = quota;
-+
-+ __refill_cfs_bandwidth_runtime(cfs_b);
-+ /* restart the period timer (if active) to handle new period expiry */
-+ if (runtime_enabled && cfs_b->timer_active) {
-+ /* force a reprogram */
-+ __start_cfs_bandwidth(cfs_b, true);
-+ }
-+ raw_spin_unlock_irq(&cfs_b->lock);
-+
-+ for_each_online_cpu(i) {
-+ struct cfs_rq *cfs_rq = tg->cfs_rq[i];
-+ struct rq *rq = cfs_rq->rq;
-+
-+ raw_spin_lock_irq(&rq->lock);
-+ cfs_rq->runtime_enabled = runtime_enabled;
-+ cfs_rq->runtime_remaining = 0;
-+
-+ if (cfs_rq->throttled)
-+ unthrottle_cfs_rq(cfs_rq);
-+ raw_spin_unlock_irq(&rq->lock);
-+ }
-+ if (runtime_was_enabled && !runtime_enabled)
-+ cfs_bandwidth_usage_dec();
-+out_unlock:
-+ mutex_unlock(&cfs_constraints_mutex);
-+ put_online_cpus();
-+
-+ return ret;
-+}
-+
-+int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
-+{
-+ u64 quota, period;
-+
-+ period = ktime_to_ns(tg->cfs_bandwidth.period);
-+ if (cfs_quota_us < 0)
-+ quota = RUNTIME_INF;
-+ else
-+ quota = (u64)cfs_quota_us * NSEC_PER_USEC;
-+
-+ return tg_set_cfs_bandwidth(tg, period, quota);
-+}
-+
-+long tg_get_cfs_quota(struct task_group *tg)
-+{
-+ u64 quota_us;
-+
-+ if (tg->cfs_bandwidth.quota == RUNTIME_INF)
-+ return -1;
-+
-+ quota_us = tg->cfs_bandwidth.quota;
-+ do_div(quota_us, NSEC_PER_USEC);
-+
-+ return quota_us;
-+}
-+
-+int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
-+{
-+ u64 quota, period;
-+
-+ period = (u64)cfs_period_us * NSEC_PER_USEC;
-+ quota = tg->cfs_bandwidth.quota;
-+
-+ return tg_set_cfs_bandwidth(tg, period, quota);
-+}
-+
-+long tg_get_cfs_period(struct task_group *tg)
-+{
-+ u64 cfs_period_us;
-+
-+ cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
-+ do_div(cfs_period_us, NSEC_PER_USEC);
-+
-+ return cfs_period_us;
-+}
-+
-+static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ return tg_get_cfs_quota(css_tg(css));
-+}
-+
-+static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
-+ struct cftype *cftype, s64 cfs_quota_us)
-+{
-+ return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
-+}
-+
-+static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ return tg_get_cfs_period(css_tg(css));
-+}
-+
-+static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
-+ struct cftype *cftype, u64 cfs_period_us)
-+{
-+ return tg_set_cfs_period(css_tg(css), cfs_period_us);
-+}
-+
-+struct cfs_schedulable_data {
-+ struct task_group *tg;
-+ u64 period, quota;
-+};
-+
-+/*
-+ * normalize group quota/period to be quota/max_period
-+ * note: units are usecs
-+ */
-+static u64 normalize_cfs_quota(struct task_group *tg,
-+ struct cfs_schedulable_data *d)
-+{
-+ u64 quota, period;
-+
-+ if (tg == d->tg) {
-+ period = d->period;
-+ quota = d->quota;
-+ } else {
-+ period = tg_get_cfs_period(tg);
-+ quota = tg_get_cfs_quota(tg);
-+ }
-+
-+ /* note: these should typically be equivalent */
-+ if (quota == RUNTIME_INF || quota == -1)
-+ return RUNTIME_INF;
-+
-+ return to_ratio(period, quota);
-+}
-+
-+static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
-+{
-+ struct cfs_schedulable_data *d = data;
-+ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
-+ s64 quota = 0, parent_quota = -1;
-+
-+ if (!tg->parent) {
-+ quota = RUNTIME_INF;
-+ } else {
-+ struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
-+
-+ quota = normalize_cfs_quota(tg, d);
-+ parent_quota = parent_b->hierarchical_quota;
-+
-+ /*
-+ * ensure max(child_quota) <= parent_quota, inherit when no
-+ * limit is set
-+ */
-+ if (quota == RUNTIME_INF)
-+ quota = parent_quota;
-+ else if (parent_quota != RUNTIME_INF && quota > parent_quota)
-+ return -EINVAL;
-+ }
-+ cfs_b->hierarchical_quota = quota;
-+
-+ return 0;
-+}
-+
-+static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
-+{
-+ int ret;
-+ struct cfs_schedulable_data data = {
-+ .tg = tg,
-+ .period = period,
-+ .quota = quota,
-+ };
-+
-+ if (quota != RUNTIME_INF) {
-+ do_div(data.period, NSEC_PER_USEC);
-+ do_div(data.quota, NSEC_PER_USEC);
-+ }
-+
-+ rcu_read_lock();
-+ ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
-+ rcu_read_unlock();
-+
-+ return ret;
-+}
-+
-+static int cpu_stats_show(struct seq_file *sf, void *v)
-+{
-+ struct task_group *tg = css_tg(seq_css(sf));
-+ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
-+
-+ seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
-+ seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
-+ seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
-+
-+ return 0;
-+}
-+#endif /* CONFIG_CFS_BANDWIDTH */
-+#endif /* CONFIG_FAIR_GROUP_SCHED */
-+
-+#ifdef CONFIG_RT_GROUP_SCHED
-+static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
-+ struct cftype *cft, s64 val)
-+{
-+ return sched_group_set_rt_runtime(css_tg(css), val);
-+}
-+
-+static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ return sched_group_rt_runtime(css_tg(css));
-+}
-+
-+static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
-+ struct cftype *cftype, u64 rt_period_us)
-+{
-+ return sched_group_set_rt_period(css_tg(css), rt_period_us);
-+}
-+
-+static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
-+ struct cftype *cft)
-+{
-+ return sched_group_rt_period(css_tg(css));
-+}
-+#endif /* CONFIG_RT_GROUP_SCHED */
-+
-+static struct cftype cpu_files[] = {
-+#ifdef CONFIG_FAIR_GROUP_SCHED
-+ {
-+ .name = "shares",
-+ .read_u64 = cpu_shares_read_u64,
-+ .write_u64 = cpu_shares_write_u64,
-+ },
-+#endif
-+#ifdef CONFIG_CFS_BANDWIDTH
-+ {
-+ .name = "cfs_quota_us",
-+ .read_s64 = cpu_cfs_quota_read_s64,
-+ .write_s64 = cpu_cfs_quota_write_s64,
-+ },
-+ {
-+ .name = "cfs_period_us",
-+ .read_u64 = cpu_cfs_period_read_u64,
-+ .write_u64 = cpu_cfs_period_write_u64,
-+ },
-+ {
-+ .name = "stat",
-+ .seq_show = cpu_stats_show,
-+ },
-+#endif
-+#ifdef CONFIG_RT_GROUP_SCHED
-+ {
-+ .name = "rt_runtime_us",
-+ .read_s64 = cpu_rt_runtime_read,
-+ .write_s64 = cpu_rt_runtime_write,
-+ },
-+ {
-+ .name = "rt_period_us",
-+ .read_u64 = cpu_rt_period_read_uint,
-+ .write_u64 = cpu_rt_period_write_uint,
-+ },
-+#endif
-+ { } /* terminate */
-+};
-+
-+struct cgroup_subsys cpu_cgrp_subsys = {
-+ .css_alloc = cpu_cgroup_css_alloc,
-+ .css_free = cpu_cgroup_css_free,
-+ .css_online = cpu_cgroup_css_online,
-+ .css_offline = cpu_cgroup_css_offline,
-+ .fork = cpu_cgroup_fork,
-+ .can_attach = cpu_cgroup_can_attach,
-+ .attach = cpu_cgroup_attach,
-+ .exit = cpu_cgroup_exit,
-+ .legacy_cftypes = cpu_files,
-+ .early_init = 1,
-+};
-+
-+#endif /* CONFIG_CGROUP_SCHED */
-+
-+void dump_cpu_task(int cpu)
-+{
-+ pr_info("Task dump for CPU %d:\n", cpu);
-+ sched_show_task(cpu_curr(cpu));
-+}
diff -Nur linux-4.1.10.orig/kernel/sched/cputime.c linux-4.1.10/kernel/sched/cputime.c
--- linux-4.1.10.orig/kernel/sched/cputime.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/cputime.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/cputime.c 2015-10-12 22:33:32.308674640 +0200
@@ -675,37 +675,45 @@
void vtime_account_system(struct task_struct *tsk)
@@ -42800,7 +19001,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/cputime.c linux-4.1.10/kernel/sched/cpu
diff -Nur linux-4.1.10.orig/kernel/sched/deadline.c linux-4.1.10/kernel/sched/deadline.c
--- linux-4.1.10.orig/kernel/sched/deadline.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/deadline.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/deadline.c 2015-10-12 22:33:32.308674640 +0200
@@ -637,6 +637,7 @@
hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
@@ -42811,7 +19012,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/deadline.c linux-4.1.10/kernel/sched/de
static
diff -Nur linux-4.1.10.orig/kernel/sched/debug.c linux-4.1.10/kernel/sched/debug.c
--- linux-4.1.10.orig/kernel/sched/debug.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/debug.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/debug.c 2015-10-12 22:33:32.308674640 +0200
@@ -260,6 +260,9 @@
P(rt_throttled);
PN(rt_time);
@@ -42835,7 +19036,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/debug.c linux-4.1.10/kernel/sched/debug
#undef P
diff -Nur linux-4.1.10.orig/kernel/sched/fair.c linux-4.1.10/kernel/sched/fair.c
--- linux-4.1.10.orig/kernel/sched/fair.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/fair.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/fair.c 2015-10-12 22:33:32.308674640 +0200
@@ -3201,7 +3201,7 @@
ideal_runtime = sched_slice(cfs_rq, curr);
delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
@@ -42910,7 +19111,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/fair.c linux-4.1.10/kernel/sched/fair.c
}
diff -Nur linux-4.1.10.orig/kernel/sched/features.h linux-4.1.10/kernel/sched/features.h
--- linux-4.1.10.orig/kernel/sched/features.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/features.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/features.h 2015-10-12 22:33:32.308674640 +0200
@@ -50,11 +50,19 @@
*/
SCHED_FEAT(NONTASK_CAPACITY, true)
@@ -42933,7 +19134,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/features.h linux-4.1.10/kernel/sched/fe
/*
diff -Nur linux-4.1.10.orig/kernel/sched/Makefile linux-4.1.10/kernel/sched/Makefile
--- linux-4.1.10.orig/kernel/sched/Makefile 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/Makefile 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/Makefile 2015-10-12 22:33:32.308674640 +0200
@@ -13,7 +13,7 @@
obj-y += core.o proc.o clock.o cputime.o
@@ -42945,7 +19146,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/Makefile linux-4.1.10/kernel/sched/Make
obj-$(CONFIG_SCHEDSTATS) += stats.o
diff -Nur linux-4.1.10.orig/kernel/sched/rt.c linux-4.1.10/kernel/sched/rt.c
--- linux-4.1.10.orig/kernel/sched/rt.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/rt.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/rt.c 2015-10-12 22:33:32.312674375 +0200
@@ -44,6 +44,7 @@
hrtimer_init(&rt_b->rt_period_timer,
@@ -42964,7 +19165,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/rt.c linux-4.1.10/kernel/sched/rt.c
/* We start is dequeued state, because no RT tasks are queued */
diff -Nur linux-4.1.10.orig/kernel/sched/sched.h linux-4.1.10/kernel/sched/sched.h
--- linux-4.1.10.orig/kernel/sched/sched.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/sched/sched.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/sched.h 2015-10-12 22:33:32.312674375 +0200
@@ -1092,6 +1092,7 @@
#define WF_SYNC 0x01 /* waker goes to sleep after wakeup */
#define WF_FORK 0x02 /* child wakeup after fork */
@@ -42991,7 +19192,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/sched.h linux-4.1.10/kernel/sched/sched
diff -Nur linux-4.1.10.orig/kernel/sched/wait-simple.c linux-4.1.10/kernel/sched/wait-simple.c
--- linux-4.1.10.orig/kernel/sched/wait-simple.c 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/kernel/sched/wait-simple.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/wait-simple.c 2015-10-12 22:33:32.312674375 +0200
@@ -0,0 +1,115 @@
+/*
+ * Simple waitqueues without fancy flags and callbacks
@@ -43110,7 +19311,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/wait-simple.c linux-4.1.10/kernel/sched
+EXPORT_SYMBOL(__swait_wake);
diff -Nur linux-4.1.10.orig/kernel/sched/work-simple.c linux-4.1.10/kernel/sched/work-simple.c
--- linux-4.1.10.orig/kernel/sched/work-simple.c 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/kernel/sched/work-simple.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/sched/work-simple.c 2015-10-12 22:33:32.312674375 +0200
@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2014 BMW Car IT GmbH, Daniel Wagner daniel.wagner@bmw-carit.de
@@ -43286,7 +19487,7 @@ diff -Nur linux-4.1.10.orig/kernel/sched/work-simple.c linux-4.1.10/kernel/sched
+EXPORT_SYMBOL_GPL(swork_put);
diff -Nur linux-4.1.10.orig/kernel/signal.c linux-4.1.10/kernel/signal.c
--- linux-4.1.10.orig/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/signal.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/signal.c 2015-10-12 22:33:32.312674375 +0200
@@ -14,6 +14,7 @@
#include <linux/export.h>
#include <linux/init.h>
@@ -43533,7 +19734,7 @@ diff -Nur linux-4.1.10.orig/kernel/signal.c linux-4.1.10/kernel/signal.c
/*
diff -Nur linux-4.1.10.orig/kernel/softirq.c linux-4.1.10/kernel/softirq.c
--- linux-4.1.10.orig/kernel/softirq.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/softirq.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/softirq.c 2015-10-12 22:33:32.312674375 +0200
@@ -21,10 +21,12 @@
#include <linux/freezer.h>
#include <linux/kthread.h>
@@ -44422,7 +20623,7 @@ diff -Nur linux-4.1.10.orig/kernel/softirq.c linux-4.1.10/kernel/softirq.c
.thread_comm = "ksoftirqd/%u",
diff -Nur linux-4.1.10.orig/kernel/stop_machine.c linux-4.1.10/kernel/stop_machine.c
--- linux-4.1.10.orig/kernel/stop_machine.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/stop_machine.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/stop_machine.c 2015-10-12 22:33:32.312674375 +0200
@@ -30,12 +30,12 @@
atomic_t nr_todo; /* nr left to execute */
bool executed; /* actually executed? */
@@ -44685,7 +20886,7 @@ diff -Nur linux-4.1.10.orig/kernel/stop_machine.c linux-4.1.10/kernel/stop_machi
mutex_unlock(&stop_cpus_mutex);
diff -Nur linux-4.1.10.orig/kernel/time/hrtimer.c linux-4.1.10/kernel/time/hrtimer.c
--- linux-4.1.10.orig/kernel/time/hrtimer.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/hrtimer.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/hrtimer.c 2015-10-12 22:33:32.312674375 +0200
@@ -48,11 +48,13 @@
#include <linux/sched/rt.h>
#include <linux/sched/deadline.h>
@@ -45292,7 +21493,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/hrtimer.c linux-4.1.10/kernel/time/hrtim
/**
diff -Nur linux-4.1.10.orig/kernel/time/itimer.c linux-4.1.10/kernel/time/itimer.c
--- linux-4.1.10.orig/kernel/time/itimer.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/itimer.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/itimer.c 2015-10-12 22:33:32.312674375 +0200
@@ -213,6 +213,7 @@
/* We are sharing ->siglock with it_real_fn() */
if (hrtimer_try_to_cancel(timer) < 0) {
@@ -45303,7 +21504,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/itimer.c linux-4.1.10/kernel/time/itimer
expires = timeval_to_ktime(value->it_value);
diff -Nur linux-4.1.10.orig/kernel/time/jiffies.c linux-4.1.10/kernel/time/jiffies.c
--- linux-4.1.10.orig/kernel/time/jiffies.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/jiffies.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/jiffies.c 2015-10-12 22:33:32.312674375 +0200
@@ -74,7 +74,8 @@
.max_cycles = 10,
};
@@ -45328,7 +21529,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/jiffies.c linux-4.1.10/kernel/time/jiffi
EXPORT_SYMBOL(get_jiffies_64);
diff -Nur linux-4.1.10.orig/kernel/time/ntp.c linux-4.1.10/kernel/time/ntp.c
--- linux-4.1.10.orig/kernel/time/ntp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/ntp.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/ntp.c 2015-10-12 22:33:32.312674375 +0200
@@ -10,6 +10,7 @@
#include <linux/workqueue.h>
#include <linux/hrtimer.h>
@@ -45392,7 +21593,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/ntp.c linux-4.1.10/kernel/time/ntp.c
void ntp_notify_cmos_timer(void) { }
diff -Nur linux-4.1.10.orig/kernel/time/posix-cpu-timers.c linux-4.1.10/kernel/time/posix-cpu-timers.c
--- linux-4.1.10.orig/kernel/time/posix-cpu-timers.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/posix-cpu-timers.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/posix-cpu-timers.c 2015-10-12 22:33:32.316674111 +0200
@@ -3,6 +3,7 @@
*/
@@ -45642,7 +21843,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/posix-cpu-timers.c linux-4.1.10/kernel/t
* The tsk->sighand->siglock must be held by the caller.
diff -Nur linux-4.1.10.orig/kernel/time/posix-timers.c linux-4.1.10/kernel/time/posix-timers.c
--- linux-4.1.10.orig/kernel/time/posix-timers.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/posix-timers.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/posix-timers.c 2015-10-12 22:33:32.316674111 +0200
@@ -499,6 +499,7 @@
static struct pid *good_sigevent(sigevent_t * event)
{
@@ -45740,7 +21941,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/posix-timers.c linux-4.1.10/kernel/time/
list_del(&timer->list);
diff -Nur linux-4.1.10.orig/kernel/time/tick-common.c linux-4.1.10/kernel/time/tick-common.c
--- linux-4.1.10.orig/kernel/time/tick-common.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/tick-common.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/tick-common.c 2015-10-12 22:33:32.316674111 +0200
@@ -78,13 +78,15 @@
static void tick_periodic(int cpu)
{
@@ -45773,7 +21974,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/tick-common.c linux-4.1.10/kernel/time/t
diff -Nur linux-4.1.10.orig/kernel/time/tick-sched.c linux-4.1.10/kernel/time/tick-sched.c
--- linux-4.1.10.orig/kernel/time/tick-sched.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/tick-sched.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/tick-sched.c 2015-10-12 22:33:32.316674111 +0200
@@ -62,7 +62,8 @@
return;
@@ -45875,7 +22076,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/tick-sched.c linux-4.1.10/kernel/time/ti
/* Get the next period (per cpu) */
diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.c linux-4.1.10/kernel/time/timekeeping.c
--- linux-4.1.10.orig/kernel/time/timekeeping.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/timekeeping.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/timekeeping.c 2015-10-12 22:33:32.316674111 +0200
@@ -2065,8 +2065,10 @@
*/
void xtime_update(unsigned long ticks)
@@ -45891,7 +22092,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.c linux-4.1.10/kernel/time/t
}
diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.h linux-4.1.10/kernel/time/timekeeping.h
--- linux-4.1.10.orig/kernel/time/timekeeping.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/timekeeping.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/timekeeping.h 2015-10-12 22:33:32.316674111 +0200
@@ -22,7 +22,8 @@
extern void do_timer(unsigned long ticks);
extern void update_wall_time(void);
@@ -45904,7 +22105,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.h linux-4.1.10/kernel/time/t
diff -Nur linux-4.1.10.orig/kernel/time/timer.c linux-4.1.10/kernel/time/timer.c
--- linux-4.1.10.orig/kernel/time/timer.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/time/timer.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/time/timer.c 2015-10-12 22:33:32.316674111 +0200
@@ -78,6 +78,9 @@
struct tvec_base {
spinlock_t lock;
@@ -46108,7 +22309,7 @@ diff -Nur linux-4.1.10.orig/kernel/time/timer.c linux-4.1.10/kernel/time/timer.c
INIT_LIST_HEAD(base->tv5.vec + j);
diff -Nur linux-4.1.10.orig/kernel/trace/Kconfig linux-4.1.10/kernel/trace/Kconfig
--- linux-4.1.10.orig/kernel/trace/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/trace/Kconfig 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/trace/Kconfig 2015-10-12 22:33:32.316674111 +0200
@@ -187,6 +187,24 @@
enabled. This option and the preempt-off timing option can be
used together or separately.)
@@ -46236,7 +22437,7 @@ diff -Nur linux-4.1.10.orig/kernel/trace/Kconfig linux-4.1.10/kernel/trace/Kconf
depends on !GENERIC_TRACER
diff -Nur linux-4.1.10.orig/kernel/trace/latency_hist.c linux-4.1.10/kernel/trace/latency_hist.c
--- linux-4.1.10.orig/kernel/trace/latency_hist.c 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/kernel/trace/latency_hist.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/trace/latency_hist.c 2015-10-12 22:33:32.316674111 +0200
@@ -0,0 +1,1178 @@
+/*
+ * kernel/trace/latency_hist.c
@@ -47418,7 +23619,7 @@ diff -Nur linux-4.1.10.orig/kernel/trace/latency_hist.c linux-4.1.10/kernel/trac
+device_initcall(latency_hist_init);
diff -Nur linux-4.1.10.orig/kernel/trace/Makefile linux-4.1.10/kernel/trace/Makefile
--- linux-4.1.10.orig/kernel/trace/Makefile 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/trace/Makefile 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/trace/Makefile 2015-10-12 22:33:32.316674111 +0200
@@ -36,6 +36,10 @@
obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o
obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o
@@ -47432,7 +23633,7 @@ diff -Nur linux-4.1.10.orig/kernel/trace/Makefile linux-4.1.10/kernel/trace/Make
obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o
diff -Nur linux-4.1.10.orig/kernel/trace/trace.c linux-4.1.10/kernel/trace/trace.c
--- linux-4.1.10.orig/kernel/trace/trace.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/trace/trace.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/trace/trace.c 2015-10-12 22:33:32.316674111 +0200
@@ -1630,6 +1630,7 @@
struct task_struct *tsk = current;
@@ -47502,7 +23703,7 @@ diff -Nur linux-4.1.10.orig/kernel/trace/trace.c linux-4.1.10/kernel/trace/trace
void
diff -Nur linux-4.1.10.orig/kernel/trace/trace_events.c linux-4.1.10/kernel/trace/trace_events.c
--- linux-4.1.10.orig/kernel/trace/trace_events.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/trace/trace_events.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/trace/trace_events.c 2015-10-12 22:33:32.320673847 +0200
@@ -162,6 +162,8 @@
__common_field(unsigned char, flags);
__common_field(unsigned char, preempt_count);
@@ -47514,7 +23715,7 @@ diff -Nur linux-4.1.10.orig/kernel/trace/trace_events.c linux-4.1.10/kernel/trac
}
diff -Nur linux-4.1.10.orig/kernel/trace/trace.h linux-4.1.10/kernel/trace/trace.h
--- linux-4.1.10.orig/kernel/trace/trace.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/trace/trace.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/trace/trace.h 2015-10-12 22:33:32.320673847 +0200
@@ -120,6 +120,7 @@
* NEED_RESCHED - reschedule is requested
* HARDIRQ - inside an interrupt handler
@@ -47533,7 +23734,7 @@ diff -Nur linux-4.1.10.orig/kernel/trace/trace.h linux-4.1.10/kernel/trace/trace
#define TRACE_BUF_SIZE 1024
diff -Nur linux-4.1.10.orig/kernel/trace/trace_irqsoff.c linux-4.1.10/kernel/trace/trace_irqsoff.c
--- linux-4.1.10.orig/kernel/trace/trace_irqsoff.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/trace/trace_irqsoff.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/trace/trace_irqsoff.c 2015-10-12 22:33:32.320673847 +0200
@@ -13,6 +13,7 @@
#include <linux/uaccess.h>
#include <linux/module.h>
@@ -47619,7 +23820,7 @@ diff -Nur linux-4.1.10.orig/kernel/trace/trace_irqsoff.c linux-4.1.10/kernel/tra
}
diff -Nur linux-4.1.10.orig/kernel/trace/trace_output.c linux-4.1.10/kernel/trace/trace_output.c
--- linux-4.1.10.orig/kernel/trace/trace_output.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/trace/trace_output.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/trace/trace_output.c 2015-10-12 22:33:32.320673847 +0200
@@ -430,6 +430,7 @@
{
char hardsoft_irq;
@@ -47667,7 +23868,7 @@ diff -Nur linux-4.1.10.orig/kernel/trace/trace_output.c linux-4.1.10/kernel/trac
diff -Nur linux-4.1.10.orig/kernel/user.c linux-4.1.10/kernel/user.c
--- linux-4.1.10.orig/kernel/user.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/user.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/user.c 2015-10-12 22:33:32.320673847 +0200
@@ -161,11 +161,11 @@
if (!up)
return;
@@ -47684,7 +23885,7 @@ diff -Nur linux-4.1.10.orig/kernel/user.c linux-4.1.10/kernel/user.c
struct user_struct *alloc_uid(kuid_t uid)
diff -Nur linux-4.1.10.orig/kernel/watchdog.c linux-4.1.10/kernel/watchdog.c
--- linux-4.1.10.orig/kernel/watchdog.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/watchdog.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/watchdog.c 2015-10-12 22:33:32.320673847 +0200
@@ -262,6 +262,8 @@
#ifdef CONFIG_HARDLOCKUP_DETECTOR
@@ -47728,7 +23929,7 @@ diff -Nur linux-4.1.10.orig/kernel/watchdog.c linux-4.1.10/kernel/watchdog.c
watchdog_nmi_enable(cpu);
diff -Nur linux-4.1.10.orig/kernel/workqueue.c linux-4.1.10/kernel/workqueue.c
--- linux-4.1.10.orig/kernel/workqueue.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/workqueue.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/workqueue.c 2015-10-12 22:33:32.320673847 +0200
@@ -48,6 +48,8 @@
#include <linux/nodemask.h>
#include <linux/moduleparam.h>
@@ -48362,7 +24563,7 @@ diff -Nur linux-4.1.10.orig/kernel/workqueue.c linux-4.1.10/kernel/workqueue.c
}
diff -Nur linux-4.1.10.orig/kernel/workqueue_internal.h linux-4.1.10/kernel/workqueue_internal.h
--- linux-4.1.10.orig/kernel/workqueue_internal.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/kernel/workqueue_internal.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/kernel/workqueue_internal.h 2015-10-12 22:33:32.320673847 +0200
@@ -43,6 +43,7 @@
unsigned long last_active; /* L: last active timestamp */
unsigned int flags; /* X: flags */
@@ -48383,7 +24584,7 @@ diff -Nur linux-4.1.10.orig/kernel/workqueue_internal.h linux-4.1.10/kernel/work
#endif /* _KERNEL_WORKQUEUE_INTERNAL_H */
diff -Nur linux-4.1.10.orig/lib/debugobjects.c linux-4.1.10/lib/debugobjects.c
--- linux-4.1.10.orig/lib/debugobjects.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/debugobjects.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/debugobjects.c 2015-10-12 22:33:32.320673847 +0200
@@ -309,7 +309,10 @@
struct debug_obj *obj;
unsigned long flags;
@@ -48398,7 +24599,7 @@ diff -Nur linux-4.1.10.orig/lib/debugobjects.c linux-4.1.10/lib/debugobjects.c
diff -Nur linux-4.1.10.orig/lib/dump_stack.c linux-4.1.10/lib/dump_stack.c
--- linux-4.1.10.orig/lib/dump_stack.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/dump_stack.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/dump_stack.c 2015-10-12 22:33:32.320673847 +0200
@@ -33,7 +33,7 @@
* Permit this cpu to perform nested stack dumps while serialising
* against other CPUs
@@ -48419,7 +24620,7 @@ diff -Nur linux-4.1.10.orig/lib/dump_stack.c linux-4.1.10/lib/dump_stack.c
asmlinkage __visible void dump_stack(void)
diff -Nur linux-4.1.10.orig/lib/idr.c linux-4.1.10/lib/idr.c
--- linux-4.1.10.orig/lib/idr.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/idr.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/idr.c 2015-10-12 22:33:32.324673582 +0200
@@ -30,6 +30,7 @@
#include <linux/idr.h>
#include <linux/spinlock.h>
@@ -48487,7 +24688,7 @@ diff -Nur linux-4.1.10.orig/lib/idr.c linux-4.1.10/lib/idr.c
diff -Nur linux-4.1.10.orig/lib/Kconfig linux-4.1.10/lib/Kconfig
--- linux-4.1.10.orig/lib/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/Kconfig 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/Kconfig 2015-10-12 22:33:32.324673582 +0200
@@ -391,6 +391,7 @@
config CPUMASK_OFFSTACK
@@ -48498,7 +24699,7 @@ diff -Nur linux-4.1.10.orig/lib/Kconfig linux-4.1.10/lib/Kconfig
them on the stack. This is a bit more expensive, but avoids
diff -Nur linux-4.1.10.orig/lib/locking-selftest.c linux-4.1.10/lib/locking-selftest.c
--- linux-4.1.10.orig/lib/locking-selftest.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/locking-selftest.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/locking-selftest.c 2015-10-12 22:33:32.324673582 +0200
@@ -590,6 +590,8 @@
#include "locking-selftest-spin-hardirq.h"
GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin)
@@ -48649,7 +24850,7 @@ diff -Nur linux-4.1.10.orig/lib/locking-selftest.c linux-4.1.10/lib/locking-self
diff -Nur linux-4.1.10.orig/lib/percpu_ida.c linux-4.1.10/lib/percpu_ida.c
--- linux-4.1.10.orig/lib/percpu_ida.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/percpu_ida.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/percpu_ida.c 2015-10-12 22:33:32.324673582 +0200
@@ -26,6 +26,9 @@
#include <linux/string.h>
#include <linux/spinlock.h>
@@ -48740,7 +24941,7 @@ diff -Nur linux-4.1.10.orig/lib/percpu_ida.c linux-4.1.10/lib/percpu_ida.c
EXPORT_SYMBOL_GPL(percpu_ida_for_each_free);
diff -Nur linux-4.1.10.orig/lib/radix-tree.c linux-4.1.10/lib/radix-tree.c
--- linux-4.1.10.orig/lib/radix-tree.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/radix-tree.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/radix-tree.c 2015-10-12 22:33:32.324673582 +0200
@@ -195,12 +195,13 @@
* succeed in getting a node here (and never reach
* kmem_cache_alloc)
@@ -48774,7 +24975,7 @@ diff -Nur linux-4.1.10.orig/lib/radix-tree.c linux-4.1.10/lib/radix-tree.c
* Return the maximum key which can be store into a
diff -Nur linux-4.1.10.orig/lib/scatterlist.c linux-4.1.10/lib/scatterlist.c
--- linux-4.1.10.orig/lib/scatterlist.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/scatterlist.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/scatterlist.c 2015-10-12 22:33:32.324673582 +0200
@@ -592,7 +592,7 @@
flush_kernel_dcache_page(miter->page);
@@ -48804,7 +25005,7 @@ diff -Nur linux-4.1.10.orig/lib/scatterlist.c linux-4.1.10/lib/scatterlist.c
diff -Nur linux-4.1.10.orig/lib/smp_processor_id.c linux-4.1.10/lib/smp_processor_id.c
--- linux-4.1.10.orig/lib/smp_processor_id.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/smp_processor_id.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/smp_processor_id.c 2015-10-12 22:33:32.324673582 +0200
@@ -39,8 +39,9 @@
if (!printk_ratelimit())
goto out_enable;
@@ -48819,7 +25020,7 @@ diff -Nur linux-4.1.10.orig/lib/smp_processor_id.c linux-4.1.10/lib/smp_processo
dump_stack();
diff -Nur linux-4.1.10.orig/lib/strnlen_user.c linux-4.1.10/lib/strnlen_user.c
--- linux-4.1.10.orig/lib/strnlen_user.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/lib/strnlen_user.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/lib/strnlen_user.c 2015-10-12 22:33:32.324673582 +0200
@@ -85,7 +85,8 @@
* @str: The string to measure.
* @count: Maximum count (including NUL character)
@@ -48842,7 +25043,7 @@ diff -Nur linux-4.1.10.orig/lib/strnlen_user.c linux-4.1.10/lib/strnlen_user.c
*
diff -Nur linux-4.1.10.orig/mm/compaction.c linux-4.1.10/mm/compaction.c
--- linux-4.1.10.orig/mm/compaction.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/compaction.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/compaction.c 2015-10-12 22:33:32.324673582 +0200
@@ -1406,10 +1406,12 @@
cc->migrate_pfn & ~((1UL << cc->order) - 1);
@@ -48860,7 +25061,7 @@ diff -Nur linux-4.1.10.orig/mm/compaction.c linux-4.1.10/mm/compaction.c
}
diff -Nur linux-4.1.10.orig/mm/filemap.c linux-4.1.10/mm/filemap.c
--- linux-4.1.10.orig/mm/filemap.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/filemap.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/filemap.c 2015-10-12 22:33:32.324673582 +0200
@@ -167,7 +167,9 @@
if (!workingset_node_pages(node) &&
list_empty(&node->private_list)) {
@@ -48889,7 +25090,7 @@ diff -Nur linux-4.1.10.orig/mm/filemap.c linux-4.1.10/mm/filemap.c
}
diff -Nur linux-4.1.10.orig/mm/highmem.c linux-4.1.10/mm/highmem.c
--- linux-4.1.10.orig/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/highmem.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/highmem.c 2015-10-12 22:33:32.324673582 +0200
@@ -29,10 +29,11 @@
#include <linux/kgdb.h>
#include <asm/tlbflush.h>
@@ -48916,7 +25117,7 @@ diff -Nur linux-4.1.10.orig/mm/highmem.c linux-4.1.10/mm/highmem.c
{
diff -Nur linux-4.1.10.orig/mm/Kconfig linux-4.1.10/mm/Kconfig
--- linux-4.1.10.orig/mm/Kconfig 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/Kconfig 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/Kconfig 2015-10-12 22:33:32.324673582 +0200
@@ -409,7 +409,7 @@
config TRANSPARENT_HUGEPAGE
@@ -48928,7 +25129,7 @@ diff -Nur linux-4.1.10.orig/mm/Kconfig linux-4.1.10/mm/Kconfig
Transparent Hugepages allows the kernel to use huge pages and
diff -Nur linux-4.1.10.orig/mm/memcontrol.c linux-4.1.10/mm/memcontrol.c
--- linux-4.1.10.orig/mm/memcontrol.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/memcontrol.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/memcontrol.c 2015-10-12 22:33:32.328673318 +0200
@@ -66,6 +66,8 @@
#include <net/sock.h>
#include <net/ip.h>
@@ -49051,7 +25252,7 @@ diff -Nur linux-4.1.10.orig/mm/memcontrol.c linux-4.1.10/mm/memcontrol.c
/**
diff -Nur linux-4.1.10.orig/mm/memory.c linux-4.1.10/mm/memory.c
--- linux-4.1.10.orig/mm/memory.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/memory.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/memory.c 2015-10-12 22:33:32.328673318 +0200
@@ -3743,7 +3743,7 @@
}
@@ -49090,7 +25291,7 @@ diff -Nur linux-4.1.10.orig/mm/memory.c linux-4.1.10/mm/memory.c
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
diff -Nur linux-4.1.10.orig/mm/mmu_context.c linux-4.1.10/mm/mmu_context.c
--- linux-4.1.10.orig/mm/mmu_context.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/mmu_context.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/mmu_context.c 2015-10-12 22:33:32.328673318 +0200
@@ -23,6 +23,7 @@
struct task_struct *tsk = current;
@@ -49109,7 +25310,7 @@ diff -Nur linux-4.1.10.orig/mm/mmu_context.c linux-4.1.10/mm/mmu_context.c
finish_arch_post_lock_switch();
diff -Nur linux-4.1.10.orig/mm/page_alloc.c linux-4.1.10/mm/page_alloc.c
--- linux-4.1.10.orig/mm/page_alloc.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/page_alloc.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/page_alloc.c 2015-10-12 22:33:32.328673318 +0200
@@ -60,6 +60,7 @@
#include <linux/page_ext.h>
#include <linux/hugetlb.h>
@@ -49434,6649 +25635,9 @@ diff -Nur linux-4.1.10.orig/mm/page_alloc.c linux-4.1.10/mm/page_alloc.c
}
#ifdef CONFIG_MEMORY_HOTREMOVE
-diff -Nur linux-4.1.10.orig/mm/page_alloc.c.orig linux-4.1.10/mm/page_alloc.c.orig
---- linux-4.1.10.orig/mm/page_alloc.c.orig 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/mm/page_alloc.c.orig 2015-10-03 13:49:38.000000000 +0200
-@@ -0,0 +1,6636 @@
-+/*
-+ * linux/mm/page_alloc.c
-+ *
-+ * Manages the free list, the system allocates free pages here.
-+ * Note that kmalloc() lives in slab.c
-+ *
-+ * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
-+ * Swap reorganised 29.12.95, Stephen Tweedie
-+ * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
-+ * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
-+ * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
-+ * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
-+ * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
-+ * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
-+ */
-+
-+#include <linux/stddef.h>
-+#include <linux/mm.h>
-+#include <linux/swap.h>
-+#include <linux/interrupt.h>
-+#include <linux/pagemap.h>
-+#include <linux/jiffies.h>
-+#include <linux/bootmem.h>
-+#include <linux/memblock.h>
-+#include <linux/compiler.h>
-+#include <linux/kernel.h>
-+#include <linux/kmemcheck.h>
-+#include <linux/kasan.h>
-+#include <linux/module.h>
-+#include <linux/suspend.h>
-+#include <linux/pagevec.h>
-+#include <linux/blkdev.h>
-+#include <linux/slab.h>
-+#include <linux/ratelimit.h>
-+#include <linux/oom.h>
-+#include <linux/notifier.h>
-+#include <linux/topology.h>
-+#include <linux/sysctl.h>
-+#include <linux/cpu.h>
-+#include <linux/cpuset.h>
-+#include <linux/memory_hotplug.h>
-+#include <linux/nodemask.h>
-+#include <linux/vmalloc.h>
-+#include <linux/vmstat.h>
-+#include <linux/mempolicy.h>
-+#include <linux/stop_machine.h>
-+#include <linux/sort.h>
-+#include <linux/pfn.h>
-+#include <linux/backing-dev.h>
-+#include <linux/fault-inject.h>
-+#include <linux/page-isolation.h>
-+#include <linux/page_ext.h>
-+#include <linux/debugobjects.h>
-+#include <linux/kmemleak.h>
-+#include <linux/compaction.h>
-+#include <trace/events/kmem.h>
-+#include <linux/prefetch.h>
-+#include <linux/mm_inline.h>
-+#include <linux/migrate.h>
-+#include <linux/page_ext.h>
-+#include <linux/hugetlb.h>
-+#include <linux/sched/rt.h>
-+#include <linux/page_owner.h>
-+
-+#include <asm/sections.h>
-+#include <asm/tlbflush.h>
-+#include <asm/div64.h>
-+#include "internal.h"
-+
-+/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
-+static DEFINE_MUTEX(pcp_batch_high_lock);
-+#define MIN_PERCPU_PAGELIST_FRACTION (8)
-+
-+#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
-+DEFINE_PER_CPU(int, numa_node);
-+EXPORT_PER_CPU_SYMBOL(numa_node);
-+#endif
-+
-+#ifdef CONFIG_HAVE_MEMORYLESS_NODES
-+/*
-+ * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
-+ * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
-+ * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
-+ * defined in <linux/topology.h>.
-+ */
-+DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
-+EXPORT_PER_CPU_SYMBOL(_numa_mem_);
-+int _node_numa_mem_[MAX_NUMNODES];
-+#endif
-+
-+/*
-+ * Array of node states.
-+ */
-+nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
-+ [N_POSSIBLE] = NODE_MASK_ALL,
-+ [N_ONLINE] = { { [0] = 1UL } },
-+#ifndef CONFIG_NUMA
-+ [N_NORMAL_MEMORY] = { { [0] = 1UL } },
-+#ifdef CONFIG_HIGHMEM
-+ [N_HIGH_MEMORY] = { { [0] = 1UL } },
-+#endif
-+#ifdef CONFIG_MOVABLE_NODE
-+ [N_MEMORY] = { { [0] = 1UL } },
-+#endif
-+ [N_CPU] = { { [0] = 1UL } },
-+#endif /* NUMA */
-+};
-+EXPORT_SYMBOL(node_states);
-+
-+/* Protect totalram_pages and zone->managed_pages */
-+static DEFINE_SPINLOCK(managed_page_count_lock);
-+
-+unsigned long totalram_pages __read_mostly;
-+unsigned long totalreserve_pages __read_mostly;
-+unsigned long totalcma_pages __read_mostly;
-+/*
-+ * When calculating the number of globally allowed dirty pages, there
-+ * is a certain number of per-zone reserves that should not be
-+ * considered dirtyable memory. This is the sum of those reserves
-+ * over all existing zones that contribute dirtyable memory.
-+ */
-+unsigned long dirty_balance_reserve __read_mostly;
-+
-+int percpu_pagelist_fraction;
-+gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
-+
-+#ifdef CONFIG_PM_SLEEP
-+/*
-+ * The following functions are used by the suspend/hibernate code to temporarily
-+ * change gfp_allowed_mask in order to avoid using I/O during memory allocations
-+ * while devices are suspended. To avoid races with the suspend/hibernate code,
-+ * they should always be called with pm_mutex held (gfp_allowed_mask also should
-+ * only be modified with pm_mutex held, unless the suspend/hibernate code is
-+ * guaranteed not to run in parallel with that modification).
-+ */
-+
-+static gfp_t saved_gfp_mask;
-+
-+void pm_restore_gfp_mask(void)
-+{
-+ WARN_ON(!mutex_is_locked(&pm_mutex));
-+ if (saved_gfp_mask) {
-+ gfp_allowed_mask = saved_gfp_mask;
-+ saved_gfp_mask = 0;
-+ }
-+}
-+
-+void pm_restrict_gfp_mask(void)
-+{
-+ WARN_ON(!mutex_is_locked(&pm_mutex));
-+ WARN_ON(saved_gfp_mask);
-+ saved_gfp_mask = gfp_allowed_mask;
-+ gfp_allowed_mask &= ~GFP_IOFS;
-+}
-+
-+bool pm_suspended_storage(void)
-+{
-+ if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS)
-+ return false;
-+ return true;
-+}
-+#endif /* CONFIG_PM_SLEEP */
-+
-+#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
-+int pageblock_order __read_mostly;
-+#endif
-+
-+static void __free_pages_ok(struct page *page, unsigned int order);
-+
-+/*
-+ * results with 256, 32 in the lowmem_reserve sysctl:
-+ * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
-+ * 1G machine -> (16M dma, 784M normal, 224M high)
-+ * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
-+ * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
-+ * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
-+ *
-+ * TBD: should special case ZONE_DMA32 machines here - in those we normally
-+ * don't need any ZONE_NORMAL reservation
-+ */
-+int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
-+#ifdef CONFIG_ZONE_DMA
-+ 256,
-+#endif
-+#ifdef CONFIG_ZONE_DMA32
-+ 256,
-+#endif
-+#ifdef CONFIG_HIGHMEM
-+ 32,
-+#endif
-+ 32,
-+};
-+
-+EXPORT_SYMBOL(totalram_pages);
-+
-+static char * const zone_names[MAX_NR_ZONES] = {
-+#ifdef CONFIG_ZONE_DMA
-+ "DMA",
-+#endif
-+#ifdef CONFIG_ZONE_DMA32
-+ "DMA32",
-+#endif
-+ "Normal",
-+#ifdef CONFIG_HIGHMEM
-+ "HighMem",
-+#endif
-+ "Movable",
-+};
-+
-+int min_free_kbytes = 1024;
-+int user_min_free_kbytes = -1;
-+
-+static unsigned long __meminitdata nr_kernel_pages;
-+static unsigned long __meminitdata nr_all_pages;
-+static unsigned long __meminitdata dma_reserve;
-+
-+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
-+static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
-+static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
-+static unsigned long __initdata required_kernelcore;
-+static unsigned long __initdata required_movablecore;
-+static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
-+
-+/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
-+int movable_zone;
-+EXPORT_SYMBOL(movable_zone);
-+#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
-+
-+#if MAX_NUMNODES > 1
-+int nr_node_ids __read_mostly = MAX_NUMNODES;
-+int nr_online_nodes __read_mostly = 1;
-+EXPORT_SYMBOL(nr_node_ids);
-+EXPORT_SYMBOL(nr_online_nodes);
-+#endif
-+
-+int page_group_by_mobility_disabled __read_mostly;
-+
-+void set_pageblock_migratetype(struct page *page, int migratetype)
-+{
-+ if (unlikely(page_group_by_mobility_disabled &&
-+ migratetype < MIGRATE_PCPTYPES))
-+ migratetype = MIGRATE_UNMOVABLE;
-+
-+ set_pageblock_flags_group(page, (unsigned long)migratetype,
-+ PB_migrate, PB_migrate_end);
-+}
-+
-+#ifdef CONFIG_DEBUG_VM
-+static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
-+{
-+ int ret = 0;
-+ unsigned seq;
-+ unsigned long pfn = page_to_pfn(page);
-+ unsigned long sp, start_pfn;
-+
-+ do {
-+ seq = zone_span_seqbegin(zone);
-+ start_pfn = zone->zone_start_pfn;
-+ sp = zone->spanned_pages;
-+ if (!zone_spans_pfn(zone, pfn))
-+ ret = 1;
-+ } while (zone_span_seqretry(zone, seq));
-+
-+ if (ret)
-+ pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
-+ pfn, zone_to_nid(zone), zone->name,
-+ start_pfn, start_pfn + sp);
-+
-+ return ret;
-+}
-+
-+static int page_is_consistent(struct zone *zone, struct page *page)
-+{
-+ if (!pfn_valid_within(page_to_pfn(page)))
-+ return 0;
-+ if (zone != page_zone(page))
-+ return 0;
-+
-+ return 1;
-+}
-+/*
-+ * Temporary debugging check for pages not lying within a given zone.
-+ */
-+static int bad_range(struct zone *zone, struct page *page)
-+{
-+ if (page_outside_zone_boundaries(zone, page))
-+ return 1;
-+ if (!page_is_consistent(zone, page))
-+ return 1;
-+
-+ return 0;
-+}
-+#else
-+static inline int bad_range(struct zone *zone, struct page *page)
-+{
-+ return 0;
-+}
-+#endif
-+
-+static void bad_page(struct page *page, const char *reason,
-+ unsigned long bad_flags)
-+{
-+ static unsigned long resume;
-+ static unsigned long nr_shown;
-+ static unsigned long nr_unshown;
-+
-+ /* Don't complain about poisoned pages */
-+ if (PageHWPoison(page)) {
-+ page_mapcount_reset(page); /* remove PageBuddy */
-+ return;
-+ }
-+
-+ /*
-+ * Allow a burst of 60 reports, then keep quiet for that minute;
-+ * or allow a steady drip of one report per second.
-+ */
-+ if (nr_shown == 60) {
-+ if (time_before(jiffies, resume)) {
-+ nr_unshown++;
-+ goto out;
-+ }
-+ if (nr_unshown) {
-+ printk(KERN_ALERT
-+ "BUG: Bad page state: %lu messages suppressed\n",
-+ nr_unshown);
-+ nr_unshown = 0;
-+ }
-+ nr_shown = 0;
-+ }
-+ if (nr_shown++ == 0)
-+ resume = jiffies + 60 * HZ;
-+
-+ printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
-+ current->comm, page_to_pfn(page));
-+ dump_page_badflags(page, reason, bad_flags);
-+
-+ print_modules();
-+ dump_stack();
-+out:
-+ /* Leave bad fields for debug, except PageBuddy could make trouble */
-+ page_mapcount_reset(page); /* remove PageBuddy */
-+ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
-+}
-+
-+/*
-+ * Higher-order pages are called "compound pages". They are structured thusly:
-+ *
-+ * The first PAGE_SIZE page is called the "head page".
-+ *
-+ * The remaining PAGE_SIZE pages are called "tail pages".
-+ *
-+ * All pages have PG_compound set. All tail pages have their ->first_page
-+ * pointing at the head page.
-+ *
-+ * The first tail page's ->lru.next holds the address of the compound page's
-+ * put_page() function. Its ->lru.prev holds the order of allocation.
-+ * This usage means that zero-order pages may not be compound.
-+ */
-+
-+static void free_compound_page(struct page *page)
-+{
-+ __free_pages_ok(page, compound_order(page));
-+}
-+
-+void prep_compound_page(struct page *page, unsigned long order)
-+{
-+ int i;
-+ int nr_pages = 1 << order;
-+
-+ set_compound_page_dtor(page, free_compound_page);
-+ set_compound_order(page, order);
-+ __SetPageHead(page);
-+ for (i = 1; i < nr_pages; i++) {
-+ struct page *p = page + i;
-+ set_page_count(p, 0);
-+ p->first_page = page;
-+ /* Make sure p->first_page is always valid for PageTail() */
-+ smp_wmb();
-+ __SetPageTail(p);
-+ }
-+}
-+
-+static inline void prep_zero_page(struct page *page, unsigned int order,
-+ gfp_t gfp_flags)
-+{
-+ int i;
-+
-+ /*
-+ * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
-+ * and __GFP_HIGHMEM from hard or soft interrupt context.
-+ */
-+ VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
-+ for (i = 0; i < (1 << order); i++)
-+ clear_highpage(page + i);
-+}
-+
-+#ifdef CONFIG_DEBUG_PAGEALLOC
-+unsigned int _debug_guardpage_minorder;
-+bool _debug_pagealloc_enabled __read_mostly;
-+bool _debug_guardpage_enabled __read_mostly;
-+
-+static int __init early_debug_pagealloc(char *buf)
-+{
-+ if (!buf)
-+ return -EINVAL;
-+
-+ if (strcmp(buf, "on") == 0)
-+ _debug_pagealloc_enabled = true;
-+
-+ return 0;
-+}
-+early_param("debug_pagealloc", early_debug_pagealloc);
-+
-+static bool need_debug_guardpage(void)
-+{
-+ /* If we don't use debug_pagealloc, we don't need guard page */
-+ if (!debug_pagealloc_enabled())
-+ return false;
-+
-+ return true;
-+}
-+
-+static void init_debug_guardpage(void)
-+{
-+ if (!debug_pagealloc_enabled())
-+ return;
-+
-+ _debug_guardpage_enabled = true;
-+}
-+
-+struct page_ext_operations debug_guardpage_ops = {
-+ .need = need_debug_guardpage,
-+ .init = init_debug_guardpage,
-+};
-+
-+static int __init debug_guardpage_minorder_setup(char *buf)
-+{
-+ unsigned long res;
-+
-+ if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
-+ printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
-+ return 0;
-+ }
-+ _debug_guardpage_minorder = res;
-+ printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
-+ return 0;
-+}
-+__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
-+
-+static inline void set_page_guard(struct zone *zone, struct page *page,
-+ unsigned int order, int migratetype)
-+{
-+ struct page_ext *page_ext;
-+
-+ if (!debug_guardpage_enabled())
-+ return;
-+
-+ page_ext = lookup_page_ext(page);
-+ __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
-+
-+ INIT_LIST_HEAD(&page->lru);
-+ set_page_private(page, order);
-+ /* Guard pages are not available for any usage */
-+ __mod_zone_freepage_state(zone, -(1 << order), migratetype);
-+}
-+
-+static inline void clear_page_guard(struct zone *zone, struct page *page,
-+ unsigned int order, int migratetype)
-+{
-+ struct page_ext *page_ext;
-+
-+ if (!debug_guardpage_enabled())
-+ return;
-+
-+ page_ext = lookup_page_ext(page);
-+ __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
-+
-+ set_page_private(page, 0);
-+ if (!is_migrate_isolate(migratetype))
-+ __mod_zone_freepage_state(zone, (1 << order), migratetype);
-+}
-+#else
-+struct page_ext_operations debug_guardpage_ops = { NULL, };
-+static inline void set_page_guard(struct zone *zone, struct page *page,
-+ unsigned int order, int migratetype) {}
-+static inline void clear_page_guard(struct zone *zone, struct page *page,
-+ unsigned int order, int migratetype) {}
-+#endif
-+
-+static inline void set_page_order(struct page *page, unsigned int order)
-+{
-+ set_page_private(page, order);
-+ __SetPageBuddy(page);
-+}
-+
-+static inline void rmv_page_order(struct page *page)
-+{
-+ __ClearPageBuddy(page);
-+ set_page_private(page, 0);
-+}
-+
-+/*
-+ * This function checks whether a page is free && is the buddy
-+ * we can do coalesce a page and its buddy if
-+ * (a) the buddy is not in a hole &&
-+ * (b) the buddy is in the buddy system &&
-+ * (c) a page and its buddy have the same order &&
-+ * (d) a page and its buddy are in the same zone.
-+ *
-+ * For recording whether a page is in the buddy system, we set ->_mapcount
-+ * PAGE_BUDDY_MAPCOUNT_VALUE.
-+ * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
-+ * serialized by zone->lock.
-+ *
-+ * For recording page's order, we use page_private(page).
-+ */
-+static inline int page_is_buddy(struct page *page, struct page *buddy,
-+ unsigned int order)
-+{
-+ if (!pfn_valid_within(page_to_pfn(buddy)))
-+ return 0;
-+
-+ if (page_is_guard(buddy) && page_order(buddy) == order) {
-+ if (page_zone_id(page) != page_zone_id(buddy))
-+ return 0;
-+
-+ VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
-+
-+ return 1;
-+ }
-+
-+ if (PageBuddy(buddy) && page_order(buddy) == order) {
-+ /*
-+ * zone check is done late to avoid uselessly
-+ * calculating zone/node ids for pages that could
-+ * never merge.
-+ */
-+ if (page_zone_id(page) != page_zone_id(buddy))
-+ return 0;
-+
-+ VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
-+
-+ return 1;
-+ }
-+ return 0;
-+}
-+
-+/*
-+ * Freeing function for a buddy system allocator.
-+ *
-+ * The concept of a buddy system is to maintain direct-mapped table
-+ * (containing bit values) for memory blocks of various "orders".
-+ * The bottom level table contains the map for the smallest allocatable
-+ * units of memory (here, pages), and each level above it describes
-+ * pairs of units from the levels below, hence, "buddies".
-+ * At a high level, all that happens here is marking the table entry
-+ * at the bottom level available, and propagating the changes upward
-+ * as necessary, plus some accounting needed to play nicely with other
-+ * parts of the VM system.
-+ * At each level, we keep a list of pages, which are heads of continuous
-+ * free pages of length of (1 << order) and marked with _mapcount
-+ * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
-+ * field.
-+ * So when we are allocating or freeing one, we can derive the state of the
-+ * other. That is, if we allocate a small block, and both were
-+ * free, the remainder of the region must be split into blocks.
-+ * If a block is freed, and its buddy is also free, then this
-+ * triggers coalescing into a block of larger size.
-+ *
-+ * -- nyc
-+ */
-+
-+static inline void __free_one_page(struct page *page,
-+ unsigned long pfn,
-+ struct zone *zone, unsigned int order,
-+ int migratetype)
-+{
-+ unsigned long page_idx;
-+ unsigned long combined_idx;
-+ unsigned long uninitialized_var(buddy_idx);
-+ struct page *buddy;
-+ int max_order = MAX_ORDER;
-+
-+ VM_BUG_ON(!zone_is_initialized(zone));
-+ VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
-+
-+ VM_BUG_ON(migratetype == -1);
-+ if (is_migrate_isolate(migratetype)) {
-+ /*
-+ * We restrict max order of merging to prevent merge
-+ * between freepages on isolate pageblock and normal
-+ * pageblock. Without this, pageblock isolation
-+ * could cause incorrect freepage accounting.
-+ */
-+ max_order = min(MAX_ORDER, pageblock_order + 1);
-+ } else {
-+ __mod_zone_freepage_state(zone, 1 << order, migratetype);
-+ }
-+
-+ page_idx = pfn & ((1 << max_order) - 1);
-+
-+ VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
-+ VM_BUG_ON_PAGE(bad_range(zone, page), page);
-+
-+ while (order < max_order - 1) {
-+ buddy_idx = __find_buddy_index(page_idx, order);
-+ buddy = page + (buddy_idx - page_idx);
-+ if (!page_is_buddy(page, buddy, order))
-+ break;
-+ /*
-+ * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
-+ * merge with it and move up one order.
-+ */
-+ if (page_is_guard(buddy)) {
-+ clear_page_guard(zone, buddy, order, migratetype);
-+ } else {
-+ list_del(&buddy->lru);
-+ zone->free_area[order].nr_free--;
-+ rmv_page_order(buddy);
-+ }
-+ combined_idx = buddy_idx & page_idx;
-+ page = page + (combined_idx - page_idx);
-+ page_idx = combined_idx;
-+ order++;
-+ }
-+ set_page_order(page, order);
-+
-+ /*
-+ * If this is not the largest possible page, check if the buddy
-+ * of the next-highest order is free. If it is, it's possible
-+ * that pages are being freed that will coalesce soon. In case,
-+ * that is happening, add the free page to the tail of the list
-+ * so it's less likely to be used soon and more likely to be merged
-+ * as a higher order page
-+ */
-+ if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
-+ struct page *higher_page, *higher_buddy;
-+ combined_idx = buddy_idx & page_idx;
-+ higher_page = page + (combined_idx - page_idx);
-+ buddy_idx = __find_buddy_index(combined_idx, order + 1);
-+ higher_buddy = higher_page + (buddy_idx - combined_idx);
-+ if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
-+ list_add_tail(&page->lru,
-+ &zone->free_area[order].free_list[migratetype]);
-+ goto out;
-+ }
-+ }
-+
-+ list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
-+out:
-+ zone->free_area[order].nr_free++;
-+}
-+
-+static inline int free_pages_check(struct page *page)
-+{
-+ const char *bad_reason = NULL;
-+ unsigned long bad_flags = 0;
-+
-+ if (unlikely(page_mapcount(page)))
-+ bad_reason = "nonzero mapcount";
-+ if (unlikely(page->mapping != NULL))
-+ bad_reason = "non-NULL mapping";
-+ if (unlikely(atomic_read(&page->_count) != 0))
-+ bad_reason = "nonzero _count";
-+ if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
-+ bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
-+ bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
-+ }
-+#ifdef CONFIG_MEMCG
-+ if (unlikely(page->mem_cgroup))
-+ bad_reason = "page still charged to cgroup";
-+#endif
-+ if (unlikely(bad_reason)) {
-+ bad_page(page, bad_reason, bad_flags);
-+ return 1;
-+ }
-+ page_cpupid_reset_last(page);
-+ if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
-+ page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
-+ return 0;
-+}
-+
-+/*
-+ * Frees a number of pages from the PCP lists
-+ * Assumes all pages on list are in same zone, and of same order.
-+ * count is the number of pages to free.
-+ *
-+ * If the zone was previously in an "all pages pinned" state then look to
-+ * see if this freeing clears that state.
-+ *
-+ * And clear the zone's pages_scanned counter, to hold off the "all pages are
-+ * pinned" detection logic.
-+ */
-+static void free_pcppages_bulk(struct zone *zone, int count,
-+ struct per_cpu_pages *pcp)
-+{
-+ int migratetype = 0;
-+ int batch_free = 0;
-+ int to_free = count;
-+ unsigned long nr_scanned;
-+
-+ spin_lock(&zone->lock);
-+ nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
-+ if (nr_scanned)
-+ __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
-+
-+ while (to_free) {
-+ struct page *page;
-+ struct list_head *list;
-+
-+ /*
-+ * Remove pages from lists in a round-robin fashion. A
-+ * batch_free count is maintained that is incremented when an
-+ * empty list is encountered. This is so more pages are freed
-+ * off fuller lists instead of spinning excessively around empty
-+ * lists
-+ */
-+ do {
-+ batch_free++;
-+ if (++migratetype == MIGRATE_PCPTYPES)
-+ migratetype = 0;
-+ list = &pcp->lists[migratetype];
-+ } while (list_empty(list));
-+
-+ /* This is the only non-empty list. Free them all. */
-+ if (batch_free == MIGRATE_PCPTYPES)
-+ batch_free = to_free;
-+
-+ do {
-+ int mt; /* migratetype of the to-be-freed page */
-+
-+ page = list_entry(list->prev, struct page, lru);
-+ /* must delete as __free_one_page list manipulates */
-+ list_del(&page->lru);
-+ mt = get_freepage_migratetype(page);
-+ if (unlikely(has_isolate_pageblock(zone)))
-+ mt = get_pageblock_migratetype(page);
-+
-+ /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
-+ __free_one_page(page, page_to_pfn(page), zone, 0, mt);
-+ trace_mm_page_pcpu_drain(page, 0, mt);
-+ } while (--to_free && --batch_free && !list_empty(list));
-+ }
-+ spin_unlock(&zone->lock);
-+}
-+
-+static void free_one_page(struct zone *zone,
-+ struct page *page, unsigned long pfn,
-+ unsigned int order,
-+ int migratetype)
-+{
-+ unsigned long nr_scanned;
-+ spin_lock(&zone->lock);
-+ nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
-+ if (nr_scanned)
-+ __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
-+
-+ if (unlikely(has_isolate_pageblock(zone) ||
-+ is_migrate_isolate(migratetype))) {
-+ migratetype = get_pfnblock_migratetype(page, pfn);
-+ }
-+ __free_one_page(page, pfn, zone, order, migratetype);
-+ spin_unlock(&zone->lock);
-+}
-+
-+static int free_tail_pages_check(struct page *head_page, struct page *page)
-+{
-+ if (!IS_ENABLED(CONFIG_DEBUG_VM))
-+ return 0;
-+ if (unlikely(!PageTail(page))) {
-+ bad_page(page, "PageTail not set", 0);
-+ return 1;
-+ }
-+ if (unlikely(page->first_page != head_page)) {
-+ bad_page(page, "first_page not consistent", 0);
-+ return 1;
-+ }
-+ return 0;
-+}
-+
-+static bool free_pages_prepare(struct page *page, unsigned int order)
-+{
-+ bool compound = PageCompound(page);
-+ int i, bad = 0;
-+
-+ VM_BUG_ON_PAGE(PageTail(page), page);
-+ VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
-+
-+ trace_mm_page_free(page, order);
-+ kmemcheck_free_shadow(page, order);
-+ kasan_free_pages(page, order);
-+
-+ if (PageAnon(page))
-+ page->mapping = NULL;
-+ bad += free_pages_check(page);
-+ for (i = 1; i < (1 << order); i++) {
-+ if (compound)
-+ bad += free_tail_pages_check(page, page + i);
-+ bad += free_pages_check(page + i);
-+ }
-+ if (bad)
-+ return false;
-+
-+ reset_page_owner(page, order);
-+
-+ if (!PageHighMem(page)) {
-+ debug_check_no_locks_freed(page_address(page),
-+ PAGE_SIZE << order);
-+ debug_check_no_obj_freed(page_address(page),
-+ PAGE_SIZE << order);
-+ }
-+ arch_free_page(page, order);
-+ kernel_map_pages(page, 1 << order, 0);
-+
-+ return true;
-+}
-+
-+static void __free_pages_ok(struct page *page, unsigned int order)
-+{
-+ unsigned long flags;
-+ int migratetype;
-+ unsigned long pfn = page_to_pfn(page);
-+
-+ if (!free_pages_prepare(page, order))
-+ return;
-+
-+ migratetype = get_pfnblock_migratetype(page, pfn);
-+ local_irq_save(flags);
-+ __count_vm_events(PGFREE, 1 << order);
-+ set_freepage_migratetype(page, migratetype);
-+ free_one_page(page_zone(page), page, pfn, order, migratetype);
-+ local_irq_restore(flags);
-+}
-+
-+void __init __free_pages_bootmem(struct page *page, unsigned int order)
-+{
-+ unsigned int nr_pages = 1 << order;
-+ struct page *p = page;
-+ unsigned int loop;
-+
-+ prefetchw(p);
-+ for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
-+ prefetchw(p + 1);
-+ __ClearPageReserved(p);
-+ set_page_count(p, 0);
-+ }
-+ __ClearPageReserved(p);
-+ set_page_count(p, 0);
-+
-+ page_zone(page)->managed_pages += nr_pages;
-+ set_page_refcounted(page);
-+ __free_pages(page, order);
-+}
-+
-+#ifdef CONFIG_CMA
-+/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
-+void __init init_cma_reserved_pageblock(struct page *page)
-+{
-+ unsigned i = pageblock_nr_pages;
-+ struct page *p = page;
-+
-+ do {
-+ __ClearPageReserved(p);
-+ set_page_count(p, 0);
-+ } while (++p, --i);
-+
-+ set_pageblock_migratetype(page, MIGRATE_CMA);
-+
-+ if (pageblock_order >= MAX_ORDER) {
-+ i = pageblock_nr_pages;
-+ p = page;
-+ do {
-+ set_page_refcounted(p);
-+ __free_pages(p, MAX_ORDER - 1);
-+ p += MAX_ORDER_NR_PAGES;
-+ } while (i -= MAX_ORDER_NR_PAGES);
-+ } else {
-+ set_page_refcounted(page);
-+ __free_pages(page, pageblock_order);
-+ }
-+
-+ adjust_managed_page_count(page, pageblock_nr_pages);
-+}
-+#endif
-+
-+/*
-+ * The order of subdivision here is critical for the IO subsystem.
-+ * Please do not alter this order without good reasons and regression
-+ * testing. Specifically, as large blocks of memory are subdivided,
-+ * the order in which smaller blocks are delivered depends on the order
-+ * they're subdivided in this function. This is the primary factor
-+ * influencing the order in which pages are delivered to the IO
-+ * subsystem according to empirical testing, and this is also justified
-+ * by considering the behavior of a buddy system containing a single
-+ * large block of memory acted on by a series of small allocations.
-+ * This behavior is a critical factor in sglist merging's success.
-+ *
-+ * -- nyc
-+ */
-+static inline void expand(struct zone *zone, struct page *page,
-+ int low, int high, struct free_area *area,
-+ int migratetype)
-+{
-+ unsigned long size = 1 << high;
-+
-+ while (high > low) {
-+ area--;
-+ high--;
-+ size >>= 1;
-+ VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
-+
-+ if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) &&
-+ debug_guardpage_enabled() &&
-+ high < debug_guardpage_minorder()) {
-+ /*
-+ * Mark as guard pages (or page), that will allow to
-+ * merge back to allocator when buddy will be freed.
-+ * Corresponding page table entries will not be touched,
-+ * pages will stay not present in virtual address space
-+ */
-+ set_page_guard(zone, &page[size], high, migratetype);
-+ continue;
-+ }
-+ list_add(&page[size].lru, &area->free_list[migratetype]);
-+ area->nr_free++;
-+ set_page_order(&page[size], high);
-+ }
-+}
-+
-+/*
-+ * This page is about to be returned from the page allocator
-+ */
-+static inline int check_new_page(struct page *page)
-+{
-+ const char *bad_reason = NULL;
-+ unsigned long bad_flags = 0;
-+
-+ if (unlikely(page_mapcount(page)))
-+ bad_reason = "nonzero mapcount";
-+ if (unlikely(page->mapping != NULL))
-+ bad_reason = "non-NULL mapping";
-+ if (unlikely(atomic_read(&page->_count) != 0))
-+ bad_reason = "nonzero _count";
-+ if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
-+ bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
-+ bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
-+ }
-+#ifdef CONFIG_MEMCG
-+ if (unlikely(page->mem_cgroup))
-+ bad_reason = "page still charged to cgroup";
-+#endif
-+ if (unlikely(bad_reason)) {
-+ bad_page(page, bad_reason, bad_flags);
-+ return 1;
-+ }
-+ return 0;
-+}
-+
-+static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
-+ int alloc_flags)
-+{
-+ int i;
-+
-+ for (i = 0; i < (1 << order); i++) {
-+ struct page *p = page + i;
-+ if (unlikely(check_new_page(p)))
-+ return 1;
-+ }
-+
-+ set_page_private(page, 0);
-+ set_page_refcounted(page);
-+
-+ arch_alloc_page(page, order);
-+ kernel_map_pages(page, 1 << order, 1);
-+ kasan_alloc_pages(page, order);
-+
-+ if (gfp_flags & __GFP_ZERO)
-+ prep_zero_page(page, order, gfp_flags);
-+
-+ if (order && (gfp_flags & __GFP_COMP))
-+ prep_compound_page(page, order);
-+
-+ set_page_owner(page, order, gfp_flags);
-+
-+ /*
-+ * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
-+ * allocate the page. The expectation is that the caller is taking
-+ * steps that will free more memory. The caller should avoid the page
-+ * being used for !PFMEMALLOC purposes.
-+ */
-+ if (alloc_flags & ALLOC_NO_WATERMARKS)
-+ set_page_pfmemalloc(page);
-+ else
-+ clear_page_pfmemalloc(page);
-+
-+ return 0;
-+}
-+
-+/*
-+ * Go through the free lists for the given migratetype and remove
-+ * the smallest available page from the freelists
-+ */
-+static inline
-+struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
-+ int migratetype)
-+{
-+ unsigned int current_order;
-+ struct free_area *area;
-+ struct page *page;
-+
-+ /* Find a page of the appropriate size in the preferred list */
-+ for (current_order = order; current_order < MAX_ORDER; ++current_order) {
-+ area = &(zone->free_area[current_order]);
-+ if (list_empty(&area->free_list[migratetype]))
-+ continue;
-+
-+ page = list_entry(area->free_list[migratetype].next,
-+ struct page, lru);
-+ list_del(&page->lru);
-+ rmv_page_order(page);
-+ area->nr_free--;
-+ expand(zone, page, order, current_order, area, migratetype);
-+ set_freepage_migratetype(page, migratetype);
-+ return page;
-+ }
-+
-+ return NULL;
-+}
-+
-+
-+/*
-+ * This array describes the order lists are fallen back to when
-+ * the free lists for the desirable migrate type are depleted
-+ */
-+static int fallbacks[MIGRATE_TYPES][4] = {
-+ [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
-+ [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
-+ [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
-+#ifdef CONFIG_CMA
-+ [MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */
-+#endif
-+ [MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */
-+#ifdef CONFIG_MEMORY_ISOLATION
-+ [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */
-+#endif
-+};
-+
-+#ifdef CONFIG_CMA
-+static struct page *__rmqueue_cma_fallback(struct zone *zone,
-+ unsigned int order)
-+{
-+ return __rmqueue_smallest(zone, order, MIGRATE_CMA);
-+}
-+#else
-+static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
-+ unsigned int order) { return NULL; }
-+#endif
-+
-+/*
-+ * Move the free pages in a range to the free lists of the requested type.
-+ * Note that start_page and end_pages are not aligned on a pageblock
-+ * boundary. If alignment is required, use move_freepages_block()
-+ */
-+int move_freepages(struct zone *zone,
-+ struct page *start_page, struct page *end_page,
-+ int migratetype)
-+{
-+ struct page *page;
-+ unsigned long order;
-+ int pages_moved = 0;
-+
-+#ifndef CONFIG_HOLES_IN_ZONE
-+ /*
-+ * page_zone is not safe to call in this context when
-+ * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
-+ * anyway as we check zone boundaries in move_freepages_block().
-+ * Remove at a later date when no bug reports exist related to
-+ * grouping pages by mobility
-+ */
-+ VM_BUG_ON(page_zone(start_page) != page_zone(end_page));
-+#endif
-+
-+ for (page = start_page; page <= end_page;) {
-+ /* Make sure we are not inadvertently changing nodes */
-+ VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
-+
-+ if (!pfn_valid_within(page_to_pfn(page))) {
-+ page++;
-+ continue;
-+ }
-+
-+ if (!PageBuddy(page)) {
-+ page++;
-+ continue;
-+ }
-+
-+ order = page_order(page);
-+ list_move(&page->lru,
-+ &zone->free_area[order].free_list[migratetype]);
-+ set_freepage_migratetype(page, migratetype);
-+ page += 1 << order;
-+ pages_moved += 1 << order;
-+ }
-+
-+ return pages_moved;
-+}
-+
-+int move_freepages_block(struct zone *zone, struct page *page,
-+ int migratetype)
-+{
-+ unsigned long start_pfn, end_pfn;
-+ struct page *start_page, *end_page;
-+
-+ start_pfn = page_to_pfn(page);
-+ start_pfn = start_pfn & ~(pageblock_nr_pages-1);
-+ start_page = pfn_to_page(start_pfn);
-+ end_page = start_page + pageblock_nr_pages - 1;
-+ end_pfn = start_pfn + pageblock_nr_pages - 1;
-+
-+ /* Do not cross zone boundaries */
-+ if (!zone_spans_pfn(zone, start_pfn))
-+ start_page = page;
-+ if (!zone_spans_pfn(zone, end_pfn))
-+ return 0;
-+
-+ return move_freepages(zone, start_page, end_page, migratetype);
-+}
-+
-+static void change_pageblock_range(struct page *pageblock_page,
-+ int start_order, int migratetype)
-+{
-+ int nr_pageblocks = 1 << (start_order - pageblock_order);
-+
-+ while (nr_pageblocks--) {
-+ set_pageblock_migratetype(pageblock_page, migratetype);
-+ pageblock_page += pageblock_nr_pages;
-+ }
-+}
-+
-+/*
-+ * When we are falling back to another migratetype during allocation, try to
-+ * steal extra free pages from the same pageblocks to satisfy further
-+ * allocations, instead of polluting multiple pageblocks.
-+ *
-+ * If we are stealing a relatively large buddy page, it is likely there will
-+ * be more free pages in the pageblock, so try to steal them all. For
-+ * reclaimable and unmovable allocations, we steal regardless of page size,
-+ * as fragmentation caused by those allocations polluting movable pageblocks
-+ * is worse than movable allocations stealing from unmovable and reclaimable
-+ * pageblocks.
-+ */
-+static bool can_steal_fallback(unsigned int order, int start_mt)
-+{
-+ /*
-+ * Leaving this order check is intended, although there is
-+ * relaxed order check in next check. The reason is that
-+ * we can actually steal whole pageblock if this condition met,
-+ * but, below check doesn't guarantee it and that is just heuristic
-+ * so could be changed anytime.
-+ */
-+ if (order >= pageblock_order)
-+ return true;
-+
-+ if (order >= pageblock_order / 2 ||
-+ start_mt == MIGRATE_RECLAIMABLE ||
-+ start_mt == MIGRATE_UNMOVABLE ||
-+ page_group_by_mobility_disabled)
-+ return true;
-+
-+ return false;
-+}
-+
-+/*
-+ * This function implements actual steal behaviour. If order is large enough,
-+ * we can steal whole pageblock. If not, we first move freepages in this
-+ * pageblock and check whether half of pages are moved or not. If half of
-+ * pages are moved, we can change migratetype of pageblock and permanently
-+ * use it's pages as requested migratetype in the future.
-+ */
-+static void steal_suitable_fallback(struct zone *zone, struct page *page,
-+ int start_type)
-+{
-+ int current_order = page_order(page);
-+ int pages;
-+
-+ /* Take ownership for orders >= pageblock_order */
-+ if (current_order >= pageblock_order) {
-+ change_pageblock_range(page, current_order, start_type);
-+ return;
-+ }
-+
-+ pages = move_freepages_block(zone, page, start_type);
-+
-+ /* Claim the whole block if over half of it is free */
-+ if (pages >= (1 << (pageblock_order-1)) ||
-+ page_group_by_mobility_disabled)
-+ set_pageblock_migratetype(page, start_type);
-+}
-+
-+/*
-+ * Check whether there is a suitable fallback freepage with requested order.
-+ * If only_stealable is true, this function returns fallback_mt only if
-+ * we can steal other freepages all together. This would help to reduce
-+ * fragmentation due to mixed migratetype pages in one pageblock.
-+ */
-+int find_suitable_fallback(struct free_area *area, unsigned int order,
-+ int migratetype, bool only_stealable, bool *can_steal)
-+{
-+ int i;
-+ int fallback_mt;
-+
-+ if (area->nr_free == 0)
-+ return -1;
-+
-+ *can_steal = false;
-+ for (i = 0;; i++) {
-+ fallback_mt = fallbacks[migratetype][i];
-+ if (fallback_mt == MIGRATE_RESERVE)
-+ break;
-+
-+ if (list_empty(&area->free_list[fallback_mt]))
-+ continue;
-+
-+ if (can_steal_fallback(order, migratetype))
-+ *can_steal = true;
-+
-+ if (!only_stealable)
-+ return fallback_mt;
-+
-+ if (*can_steal)
-+ return fallback_mt;
-+ }
-+
-+ return -1;
-+}
-+
-+/* Remove an element from the buddy allocator from the fallback list */
-+static inline struct page *
-+__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
-+{
-+ struct free_area *area;
-+ unsigned int current_order;
-+ struct page *page;
-+ int fallback_mt;
-+ bool can_steal;
-+
-+ /* Find the largest possible block of pages in the other list */
-+ for (current_order = MAX_ORDER-1;
-+ current_order >= order && current_order <= MAX_ORDER-1;
-+ --current_order) {
-+ area = &(zone->free_area[current_order]);
-+ fallback_mt = find_suitable_fallback(area, current_order,
-+ start_migratetype, false, &can_steal);
-+ if (fallback_mt == -1)
-+ continue;
-+
-+ page = list_entry(area->free_list[fallback_mt].next,
-+ struct page, lru);
-+ if (can_steal)
-+ steal_suitable_fallback(zone, page, start_migratetype);
-+
-+ /* Remove the page from the freelists */
-+ area->nr_free--;
-+ list_del(&page->lru);
-+ rmv_page_order(page);
-+
-+ expand(zone, page, order, current_order, area,
-+ start_migratetype);
-+ /*
-+ * The freepage_migratetype may differ from pageblock's
-+ * migratetype depending on the decisions in
-+ * try_to_steal_freepages(). This is OK as long as it
-+ * does not differ for MIGRATE_CMA pageblocks. For CMA
-+ * we need to make sure unallocated pages flushed from
-+ * pcp lists are returned to the correct freelist.
-+ */
-+ set_freepage_migratetype(page, start_migratetype);
-+
-+ trace_mm_page_alloc_extfrag(page, order, current_order,
-+ start_migratetype, fallback_mt);
-+
-+ return page;
-+ }
-+
-+ return NULL;
-+}
-+
-+/*
-+ * Do the hard work of removing an element from the buddy allocator.
-+ * Call me with the zone->lock already held.
-+ */
-+static struct page *__rmqueue(struct zone *zone, unsigned int order,
-+ int migratetype)
-+{
-+ struct page *page;
-+
-+retry_reserve:
-+ page = __rmqueue_smallest(zone, order, migratetype);
-+
-+ if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
-+ if (migratetype == MIGRATE_MOVABLE)
-+ page = __rmqueue_cma_fallback(zone, order);
-+
-+ if (!page)
-+ page = __rmqueue_fallback(zone, order, migratetype);
-+
-+ /*
-+ * Use MIGRATE_RESERVE rather than fail an allocation. goto
-+ * is used because __rmqueue_smallest is an inline function
-+ * and we want just one call site
-+ */
-+ if (!page) {
-+ migratetype = MIGRATE_RESERVE;
-+ goto retry_reserve;
-+ }
-+ }
-+
-+ trace_mm_page_alloc_zone_locked(page, order, migratetype);
-+ return page;
-+}
-+
-+/*
-+ * Obtain a specified number of elements from the buddy allocator, all under
-+ * a single hold of the lock, for efficiency. Add them to the supplied list.
-+ * Returns the number of new pages which were placed at *list.
-+ */
-+static int rmqueue_bulk(struct zone *zone, unsigned int order,
-+ unsigned long count, struct list_head *list,
-+ int migratetype, bool cold)
-+{
-+ int i;
-+
-+ spin_lock(&zone->lock);
-+ for (i = 0; i < count; ++i) {
-+ struct page *page = __rmqueue(zone, order, migratetype);
-+ if (unlikely(page == NULL))
-+ break;
-+
-+ /*
-+ * Split buddy pages returned by expand() are received here
-+ * in physical page order. The page is added to the callers and
-+ * list and the list head then moves forward. From the callers
-+ * perspective, the linked list is ordered by page number in
-+ * some conditions. This is useful for IO devices that can
-+ * merge IO requests if the physical pages are ordered
-+ * properly.
-+ */
-+ if (likely(!cold))
-+ list_add(&page->lru, list);
-+ else
-+ list_add_tail(&page->lru, list);
-+ list = &page->lru;
-+ if (is_migrate_cma(get_freepage_migratetype(page)))
-+ __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
-+ -(1 << order));
-+ }
-+ __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
-+ spin_unlock(&zone->lock);
-+ return i;
-+}
-+
-+#ifdef CONFIG_NUMA
-+/*
-+ * Called from the vmstat counter updater to drain pagesets of this
-+ * currently executing processor on remote nodes after they have
-+ * expired.
-+ *
-+ * Note that this function must be called with the thread pinned to
-+ * a single processor.
-+ */
-+void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
-+{
-+ unsigned long flags;
-+ int to_drain, batch;
-+
-+ local_irq_save(flags);
-+ batch = READ_ONCE(pcp->batch);
-+ to_drain = min(pcp->count, batch);
-+ if (to_drain > 0) {
-+ free_pcppages_bulk(zone, to_drain, pcp);
-+ pcp->count -= to_drain;
-+ }
-+ local_irq_restore(flags);
-+}
-+#endif
-+
-+/*
-+ * Drain pcplists of the indicated processor and zone.
-+ *
-+ * The processor must either be the current processor and the
-+ * thread pinned to the current processor or a processor that
-+ * is not online.
-+ */
-+static void drain_pages_zone(unsigned int cpu, struct zone *zone)
-+{
-+ unsigned long flags;
-+ struct per_cpu_pageset *pset;
-+ struct per_cpu_pages *pcp;
-+
-+ local_irq_save(flags);
-+ pset = per_cpu_ptr(zone->pageset, cpu);
-+
-+ pcp = &pset->pcp;
-+ if (pcp->count) {
-+ free_pcppages_bulk(zone, pcp->count, pcp);
-+ pcp->count = 0;
-+ }
-+ local_irq_restore(flags);
-+}
-+
-+/*
-+ * Drain pcplists of all zones on the indicated processor.
-+ *
-+ * The processor must either be the current processor and the
-+ * thread pinned to the current processor or a processor that
-+ * is not online.
-+ */
-+static void drain_pages(unsigned int cpu)
-+{
-+ struct zone *zone;
-+
-+ for_each_populated_zone(zone) {
-+ drain_pages_zone(cpu, zone);
-+ }
-+}
-+
-+/*
-+ * Spill all of this CPU's per-cpu pages back into the buddy allocator.
-+ *
-+ * The CPU has to be pinned. When zone parameter is non-NULL, spill just
-+ * the single zone's pages.
-+ */
-+void drain_local_pages(struct zone *zone)
-+{
-+ int cpu = smp_processor_id();
-+
-+ if (zone)
-+ drain_pages_zone(cpu, zone);
-+ else
-+ drain_pages(cpu);
-+}
-+
-+/*
-+ * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
-+ *
-+ * When zone parameter is non-NULL, spill just the single zone's pages.
-+ *
-+ * Note that this code is protected against sending an IPI to an offline
-+ * CPU but does not guarantee sending an IPI to newly hotplugged CPUs:
-+ * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but
-+ * nothing keeps CPUs from showing up after we populated the cpumask and
-+ * before the call to on_each_cpu_mask().
-+ */
-+void drain_all_pages(struct zone *zone)
-+{
-+ int cpu;
-+
-+ /*
-+ * Allocate in the BSS so we wont require allocation in
-+ * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
-+ */
-+ static cpumask_t cpus_with_pcps;
-+
-+ /*
-+ * We don't care about racing with CPU hotplug event
-+ * as offline notification will cause the notified
-+ * cpu to drain that CPU pcps and on_each_cpu_mask
-+ * disables preemption as part of its processing
-+ */
-+ for_each_online_cpu(cpu) {
-+ struct per_cpu_pageset *pcp;
-+ struct zone *z;
-+ bool has_pcps = false;
-+
-+ if (zone) {
-+ pcp = per_cpu_ptr(zone->pageset, cpu);
-+ if (pcp->pcp.count)
-+ has_pcps = true;
-+ } else {
-+ for_each_populated_zone(z) {
-+ pcp = per_cpu_ptr(z->pageset, cpu);
-+ if (pcp->pcp.count) {
-+ has_pcps = true;
-+ break;
-+ }
-+ }
-+ }
-+
-+ if (has_pcps)
-+ cpumask_set_cpu(cpu, &cpus_with_pcps);
-+ else
-+ cpumask_clear_cpu(cpu, &cpus_with_pcps);
-+ }
-+ on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages,
-+ zone, 1);
-+}
-+
-+#ifdef CONFIG_HIBERNATION
-+
-+void mark_free_pages(struct zone *zone)
-+{
-+ unsigned long pfn, max_zone_pfn;
-+ unsigned long flags;
-+ unsigned int order, t;
-+ struct list_head *curr;
-+
-+ if (zone_is_empty(zone))
-+ return;
-+
-+ spin_lock_irqsave(&zone->lock, flags);
-+
-+ max_zone_pfn = zone_end_pfn(zone);
-+ for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
-+ if (pfn_valid(pfn)) {
-+ struct page *page = pfn_to_page(pfn);
-+
-+ if (!swsusp_page_is_forbidden(page))
-+ swsusp_unset_page_free(page);
-+ }
-+
-+ for_each_migratetype_order(order, t) {
-+ list_for_each(curr, &zone->free_area[order].free_list[t]) {
-+ unsigned long i;
-+
-+ pfn = page_to_pfn(list_entry(curr, struct page, lru));
-+ for (i = 0; i < (1UL << order); i++)
-+ swsusp_set_page_free(pfn_to_page(pfn + i));
-+ }
-+ }
-+ spin_unlock_irqrestore(&zone->lock, flags);
-+}
-+#endif /* CONFIG_PM */
-+
-+/*
-+ * Free a 0-order page
-+ * cold == true ? free a cold page : free a hot page
-+ */
-+void free_hot_cold_page(struct page *page, bool cold)
-+{
-+ struct zone *zone = page_zone(page);
-+ struct per_cpu_pages *pcp;
-+ unsigned long flags;
-+ unsigned long pfn = page_to_pfn(page);
-+ int migratetype;
-+
-+ if (!free_pages_prepare(page, 0))
-+ return;
-+
-+ migratetype = get_pfnblock_migratetype(page, pfn);
-+ set_freepage_migratetype(page, migratetype);
-+ local_irq_save(flags);
-+ __count_vm_event(PGFREE);
-+
-+ /*
-+ * We only track unmovable, reclaimable and movable on pcp lists.
-+ * Free ISOLATE pages back to the allocator because they are being
-+ * offlined but treat RESERVE as movable pages so we can get those
-+ * areas back if necessary. Otherwise, we may have to free
-+ * excessively into the page allocator
-+ */
-+ if (migratetype >= MIGRATE_PCPTYPES) {
-+ if (unlikely(is_migrate_isolate(migratetype))) {
-+ free_one_page(zone, page, pfn, 0, migratetype);
-+ goto out;
-+ }
-+ migratetype = MIGRATE_MOVABLE;
-+ }
-+
-+ pcp = &this_cpu_ptr(zone->pageset)->pcp;
-+ if (!cold)
-+ list_add(&page->lru, &pcp->lists[migratetype]);
-+ else
-+ list_add_tail(&page->lru, &pcp->lists[migratetype]);
-+ pcp->count++;
-+ if (pcp->count >= pcp->high) {
-+ unsigned long batch = READ_ONCE(pcp->batch);
-+ free_pcppages_bulk(zone, batch, pcp);
-+ pcp->count -= batch;
-+ }
-+
-+out:
-+ local_irq_restore(flags);
-+}
-+
-+/*
-+ * Free a list of 0-order pages
-+ */
-+void free_hot_cold_page_list(struct list_head *list, bool cold)
-+{
-+ struct page *page, *next;
-+
-+ list_for_each_entry_safe(page, next, list, lru) {
-+ trace_mm_page_free_batched(page, cold);
-+ free_hot_cold_page(page, cold);
-+ }
-+}
-+
-+/*
-+ * split_page takes a non-compound higher-order page, and splits it into
-+ * n (1<<order) sub-pages: page[0..n]
-+ * Each sub-page must be freed individually.
-+ *
-+ * Note: this is probably too low level an operation for use in drivers.
-+ * Please consult with lkml before using this in your driver.
-+ */
-+void split_page(struct page *page, unsigned int order)
-+{
-+ int i;
-+
-+ VM_BUG_ON_PAGE(PageCompound(page), page);
-+ VM_BUG_ON_PAGE(!page_count(page), page);
-+
-+#ifdef CONFIG_KMEMCHECK
-+ /*
-+ * Split shadow pages too, because free(page[0]) would
-+ * otherwise free the whole shadow.
-+ */
-+ if (kmemcheck_page_is_tracked(page))
-+ split_page(virt_to_page(page[0].shadow), order);
-+#endif
-+
-+ set_page_owner(page, 0, 0);
-+ for (i = 1; i < (1 << order); i++) {
-+ set_page_refcounted(page + i);
-+ set_page_owner(page + i, 0, 0);
-+ }
-+}
-+EXPORT_SYMBOL_GPL(split_page);
-+
-+int __isolate_free_page(struct page *page, unsigned int order)
-+{
-+ unsigned long watermark;
-+ struct zone *zone;
-+ int mt;
-+
-+ BUG_ON(!PageBuddy(page));
-+
-+ zone = page_zone(page);
-+ mt = get_pageblock_migratetype(page);
-+
-+ if (!is_migrate_isolate(mt)) {
-+ /* Obey watermarks as if the page was being allocated */
-+ watermark = low_wmark_pages(zone) + (1 << order);
-+ if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
-+ return 0;
-+
-+ __mod_zone_freepage_state(zone, -(1UL << order), mt);
-+ }
-+
-+ /* Remove page from free list */
-+ list_del(&page->lru);
-+ zone->free_area[order].nr_free--;
-+ rmv_page_order(page);
-+
-+ /* Set the pageblock if the isolated page is at least a pageblock */
-+ if (order >= pageblock_order - 1) {
-+ struct page *endpage = page + (1 << order) - 1;
-+ for (; page < endpage; page += pageblock_nr_pages) {
-+ int mt = get_pageblock_migratetype(page);
-+ if (!is_migrate_isolate(mt) && !is_migrate_cma(mt))
-+ set_pageblock_migratetype(page,
-+ MIGRATE_MOVABLE);
-+ }
-+ }
-+
-+ set_page_owner(page, order, 0);
-+ return 1UL << order;
-+}
-+
-+/*
-+ * Similar to split_page except the page is already free. As this is only
-+ * being used for migration, the migratetype of the block also changes.
-+ * As this is called with interrupts disabled, the caller is responsible
-+ * for calling arch_alloc_page() and kernel_map_page() after interrupts
-+ * are enabled.
-+ *
-+ * Note: this is probably too low level an operation for use in drivers.
-+ * Please consult with lkml before using this in your driver.
-+ */
-+int split_free_page(struct page *page)
-+{
-+ unsigned int order;
-+ int nr_pages;
-+
-+ order = page_order(page);
-+
-+ nr_pages = __isolate_free_page(page, order);
-+ if (!nr_pages)
-+ return 0;
-+
-+ /* Split into individual pages */
-+ set_page_refcounted(page);
-+ split_page(page, order);
-+ return nr_pages;
-+}
-+
-+/*
-+ * Allocate a page from the given zone. Use pcplists for order-0 allocations.
-+ */
-+static inline
-+struct page *buffered_rmqueue(struct zone *preferred_zone,
-+ struct zone *zone, unsigned int order,
-+ gfp_t gfp_flags, int migratetype)
-+{
-+ unsigned long flags;
-+ struct page *page;
-+ bool cold = ((gfp_flags & __GFP_COLD) != 0);
-+
-+ if (likely(order == 0)) {
-+ struct per_cpu_pages *pcp;
-+ struct list_head *list;
-+
-+ local_irq_save(flags);
-+ pcp = &this_cpu_ptr(zone->pageset)->pcp;
-+ list = &pcp->lists[migratetype];
-+ if (list_empty(list)) {
-+ pcp->count += rmqueue_bulk(zone, 0,
-+ pcp->batch, list,
-+ migratetype, cold);
-+ if (unlikely(list_empty(list)))
-+ goto failed;
-+ }
-+
-+ if (cold)
-+ page = list_entry(list->prev, struct page, lru);
-+ else
-+ page = list_entry(list->next, struct page, lru);
-+
-+ list_del(&page->lru);
-+ pcp->count--;
-+ } else {
-+ if (unlikely(gfp_flags & __GFP_NOFAIL)) {
-+ /*
-+ * __GFP_NOFAIL is not to be used in new code.
-+ *
-+ * All __GFP_NOFAIL callers should be fixed so that they
-+ * properly detect and handle allocation failures.
-+ *
-+ * We most definitely don't want callers attempting to
-+ * allocate greater than order-1 page units with
-+ * __GFP_NOFAIL.
-+ */
-+ WARN_ON_ONCE(order > 1);
-+ }
-+ spin_lock_irqsave(&zone->lock, flags);
-+ page = __rmqueue(zone, order, migratetype);
-+ spin_unlock(&zone->lock);
-+ if (!page)
-+ goto failed;
-+ __mod_zone_freepage_state(zone, -(1 << order),
-+ get_freepage_migratetype(page));
-+ }
-+
-+ __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
-+ if (atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 &&
-+ !test_bit(ZONE_FAIR_DEPLETED, &zone->flags))
-+ set_bit(ZONE_FAIR_DEPLETED, &zone->flags);
-+
-+ __count_zone_vm_events(PGALLOC, zone, 1 << order);
-+ zone_statistics(preferred_zone, zone, gfp_flags);
-+ local_irq_restore(flags);
-+
-+ VM_BUG_ON_PAGE(bad_range(zone, page), page);
-+ return page;
-+
-+failed:
-+ local_irq_restore(flags);
-+ return NULL;
-+}
-+
-+#ifdef CONFIG_FAIL_PAGE_ALLOC
-+
-+static struct {
-+ struct fault_attr attr;
-+
-+ u32 ignore_gfp_highmem;
-+ u32 ignore_gfp_wait;
-+ u32 min_order;
-+} fail_page_alloc = {
-+ .attr = FAULT_ATTR_INITIALIZER,
-+ .ignore_gfp_wait = 1,
-+ .ignore_gfp_highmem = 1,
-+ .min_order = 1,
-+};
-+
-+static int __init setup_fail_page_alloc(char *str)
-+{
-+ return setup_fault_attr(&fail_page_alloc.attr, str);
-+}
-+__setup("fail_page_alloc=", setup_fail_page_alloc);
-+
-+static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
-+{
-+ if (order < fail_page_alloc.min_order)
-+ return false;
-+ if (gfp_mask & __GFP_NOFAIL)
-+ return false;
-+ if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
-+ return false;
-+ if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
-+ return false;
-+
-+ return should_fail(&fail_page_alloc.attr, 1 << order);
-+}
-+
-+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
-+
-+static int __init fail_page_alloc_debugfs(void)
-+{
-+ umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
-+ struct dentry *dir;
-+
-+ dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
-+ &fail_page_alloc.attr);
-+ if (IS_ERR(dir))
-+ return PTR_ERR(dir);
-+
-+ if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
-+ &fail_page_alloc.ignore_gfp_wait))
-+ goto fail;
-+ if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
-+ &fail_page_alloc.ignore_gfp_highmem))
-+ goto fail;
-+ if (!debugfs_create_u32("min-order", mode, dir,
-+ &fail_page_alloc.min_order))
-+ goto fail;
-+
-+ return 0;
-+fail:
-+ debugfs_remove_recursive(dir);
-+
-+ return -ENOMEM;
-+}
-+
-+late_initcall(fail_page_alloc_debugfs);
-+
-+#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
-+
-+#else /* CONFIG_FAIL_PAGE_ALLOC */
-+
-+static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
-+{
-+ return false;
-+}
-+
-+#endif /* CONFIG_FAIL_PAGE_ALLOC */
-+
-+/*
-+ * Return true if free pages are above 'mark'. This takes into account the order
-+ * of the allocation.
-+ */
-+static bool __zone_watermark_ok(struct zone *z, unsigned int order,
-+ unsigned long mark, int classzone_idx, int alloc_flags,
-+ long free_pages)
-+{
-+ /* free_pages may go negative - that's OK */
-+ long min = mark;
-+ int o;
-+ long free_cma = 0;
-+
-+ free_pages -= (1 << order) - 1;
-+ if (alloc_flags & ALLOC_HIGH)
-+ min -= min / 2;
-+ if (alloc_flags & ALLOC_HARDER)
-+ min -= min / 4;
-+#ifdef CONFIG_CMA
-+ /* If allocation can't use CMA areas don't use free CMA pages */
-+ if (!(alloc_flags & ALLOC_CMA))
-+ free_cma = zone_page_state(z, NR_FREE_CMA_PAGES);
-+#endif
-+
-+ if (free_pages - free_cma <= min + z->lowmem_reserve[classzone_idx])
-+ return false;
-+ for (o = 0; o < order; o++) {
-+ /* At the next order, this order's pages become unavailable */
-+ free_pages -= z->free_area[o].nr_free << o;
-+
-+ /* Require fewer higher order pages to be free */
-+ min >>= 1;
-+
-+ if (free_pages <= min)
-+ return false;
-+ }
-+ return true;
-+}
-+
-+bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
-+ int classzone_idx, int alloc_flags)
-+{
-+ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
-+ zone_page_state(z, NR_FREE_PAGES));
-+}
-+
-+bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
-+ unsigned long mark, int classzone_idx, int alloc_flags)
-+{
-+ long free_pages = zone_page_state(z, NR_FREE_PAGES);
-+
-+ if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
-+ free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
-+
-+ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
-+ free_pages);
-+}
-+
-+#ifdef CONFIG_NUMA
-+/*
-+ * zlc_setup - Setup for "zonelist cache". Uses cached zone data to
-+ * skip over zones that are not allowed by the cpuset, or that have
-+ * been recently (in last second) found to be nearly full. See further
-+ * comments in mmzone.h. Reduces cache footprint of zonelist scans
-+ * that have to skip over a lot of full or unallowed zones.
-+ *
-+ * If the zonelist cache is present in the passed zonelist, then
-+ * returns a pointer to the allowed node mask (either the current
-+ * tasks mems_allowed, or node_states[N_MEMORY].)
-+ *
-+ * If the zonelist cache is not available for this zonelist, does
-+ * nothing and returns NULL.
-+ *
-+ * If the fullzones BITMAP in the zonelist cache is stale (more than
-+ * a second since last zap'd) then we zap it out (clear its bits.)
-+ *
-+ * We hold off even calling zlc_setup, until after we've checked the
-+ * first zone in the zonelist, on the theory that most allocations will
-+ * be satisfied from that first zone, so best to examine that zone as
-+ * quickly as we can.
-+ */
-+static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
-+{
-+ struct zonelist_cache *zlc; /* cached zonelist speedup info */
-+ nodemask_t *allowednodes; /* zonelist_cache approximation */
-+
-+ zlc = zonelist->zlcache_ptr;
-+ if (!zlc)
-+ return NULL;
-+
-+ if (time_after(jiffies, zlc->last_full_zap + HZ)) {
-+ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
-+ zlc->last_full_zap = jiffies;
-+ }
-+
-+ allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
-+ &cpuset_current_mems_allowed :
-+ &node_states[N_MEMORY];
-+ return allowednodes;
-+}
-+
-+/*
-+ * Given 'z' scanning a zonelist, run a couple of quick checks to see
-+ * if it is worth looking at further for free memory:
-+ * 1) Check that the zone isn't thought to be full (doesn't have its
-+ * bit set in the zonelist_cache fullzones BITMAP).
-+ * 2) Check that the zones node (obtained from the zonelist_cache
-+ * z_to_n[] mapping) is allowed in the passed in allowednodes mask.
-+ * Return true (non-zero) if zone is worth looking at further, or
-+ * else return false (zero) if it is not.
-+ *
-+ * This check -ignores- the distinction between various watermarks,
-+ * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is
-+ * found to be full for any variation of these watermarks, it will
-+ * be considered full for up to one second by all requests, unless
-+ * we are so low on memory on all allowed nodes that we are forced
-+ * into the second scan of the zonelist.
-+ *
-+ * In the second scan we ignore this zonelist cache and exactly
-+ * apply the watermarks to all zones, even it is slower to do so.
-+ * We are low on memory in the second scan, and should leave no stone
-+ * unturned looking for a free page.
-+ */
-+static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
-+ nodemask_t *allowednodes)
-+{
-+ struct zonelist_cache *zlc; /* cached zonelist speedup info */
-+ int i; /* index of *z in zonelist zones */
-+ int n; /* node that zone *z is on */
-+
-+ zlc = zonelist->zlcache_ptr;
-+ if (!zlc)
-+ return 1;
-+
-+ i = z - zonelist->_zonerefs;
-+ n = zlc->z_to_n[i];
-+
-+ /* This zone is worth trying if it is allowed but not full */
-+ return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
-+}
-+
-+/*
-+ * Given 'z' scanning a zonelist, set the corresponding bit in
-+ * zlc->fullzones, so that subsequent attempts to allocate a page
-+ * from that zone don't waste time re-examining it.
-+ */
-+static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
-+{
-+ struct zonelist_cache *zlc; /* cached zonelist speedup info */
-+ int i; /* index of *z in zonelist zones */
-+
-+ zlc = zonelist->zlcache_ptr;
-+ if (!zlc)
-+ return;
-+
-+ i = z - zonelist->_zonerefs;
-+
-+ set_bit(i, zlc->fullzones);
-+}
-+
-+/*
-+ * clear all zones full, called after direct reclaim makes progress so that
-+ * a zone that was recently full is not skipped over for up to a second
-+ */
-+static void zlc_clear_zones_full(struct zonelist *zonelist)
-+{
-+ struct zonelist_cache *zlc; /* cached zonelist speedup info */
-+
-+ zlc = zonelist->zlcache_ptr;
-+ if (!zlc)
-+ return;
-+
-+ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
-+}
-+
-+static bool zone_local(struct zone *local_zone, struct zone *zone)
-+{
-+ return local_zone->node == zone->node;
-+}
-+
-+static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
-+{
-+ return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <
-+ RECLAIM_DISTANCE;
-+}
-+
-+#else /* CONFIG_NUMA */
-+
-+static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
-+{
-+ return NULL;
-+}
-+
-+static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
-+ nodemask_t *allowednodes)
-+{
-+ return 1;
-+}
-+
-+static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
-+{
-+}
-+
-+static void zlc_clear_zones_full(struct zonelist *zonelist)
-+{
-+}
-+
-+static bool zone_local(struct zone *local_zone, struct zone *zone)
-+{
-+ return true;
-+}
-+
-+static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
-+{
-+ return true;
-+}
-+
-+#endif /* CONFIG_NUMA */
-+
-+static void reset_alloc_batches(struct zone *preferred_zone)
-+{
-+ struct zone *zone = preferred_zone->zone_pgdat->node_zones;
-+
-+ do {
-+ mod_zone_page_state(zone, NR_ALLOC_BATCH,
-+ high_wmark_pages(zone) - low_wmark_pages(zone) -
-+ atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]));
-+ clear_bit(ZONE_FAIR_DEPLETED, &zone->flags);
-+ } while (zone++ != preferred_zone);
-+}
-+
-+/*
-+ * get_page_from_freelist goes through the zonelist trying to allocate
-+ * a page.
-+ */
-+static struct page *
-+get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
-+ const struct alloc_context *ac)
-+{
-+ struct zonelist *zonelist = ac->zonelist;
-+ struct zoneref *z;
-+ struct page *page = NULL;
-+ struct zone *zone;
-+ nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
-+ int zlc_active = 0; /* set if using zonelist_cache */
-+ int did_zlc_setup = 0; /* just call zlc_setup() one time */
-+ bool consider_zone_dirty = (alloc_flags & ALLOC_WMARK_LOW) &&
-+ (gfp_mask & __GFP_WRITE);
-+ int nr_fair_skipped = 0;
-+ bool zonelist_rescan;
-+
-+zonelist_scan:
-+ zonelist_rescan = false;
-+
-+ /*
-+ * Scan zonelist, looking for a zone with enough free.
-+ * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
-+ */
-+ for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
-+ ac->nodemask) {
-+ unsigned long mark;
-+
-+ if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&
-+ !zlc_zone_worth_trying(zonelist, z, allowednodes))
-+ continue;
-+ if (cpusets_enabled() &&
-+ (alloc_flags & ALLOC_CPUSET) &&
-+ !cpuset_zone_allowed(zone, gfp_mask))
-+ continue;
-+ /*
-+ * Distribute pages in proportion to the individual
-+ * zone size to ensure fair page aging. The zone a
-+ * page was allocated in should have no effect on the
-+ * time the page has in memory before being reclaimed.
-+ */
-+ if (alloc_flags & ALLOC_FAIR) {
-+ if (!zone_local(ac->preferred_zone, zone))
-+ break;
-+ if (test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) {
-+ nr_fair_skipped++;
-+ continue;
-+ }
-+ }
-+ /*
-+ * When allocating a page cache page for writing, we
-+ * want to get it from a zone that is within its dirty
-+ * limit, such that no single zone holds more than its
-+ * proportional share of globally allowed dirty pages.
-+ * The dirty limits take into account the zone's
-+ * lowmem reserves and high watermark so that kswapd
-+ * should be able to balance it without having to
-+ * write pages from its LRU list.
-+ *
-+ * This may look like it could increase pressure on
-+ * lower zones by failing allocations in higher zones
-+ * before they are full. But the pages that do spill
-+ * over are limited as the lower zones are protected
-+ * by this very same mechanism. It should not become
-+ * a practical burden to them.
-+ *
-+ * XXX: For now, allow allocations to potentially
-+ * exceed the per-zone dirty limit in the slowpath
-+ * (ALLOC_WMARK_LOW unset) before going into reclaim,
-+ * which is important when on a NUMA setup the allowed
-+ * zones are together not big enough to reach the
-+ * global limit. The proper fix for these situations
-+ * will require awareness of zones in the
-+ * dirty-throttling and the flusher threads.
-+ */
-+ if (consider_zone_dirty && !zone_dirty_ok(zone))
-+ continue;
-+
-+ mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
-+ if (!zone_watermark_ok(zone, order, mark,
-+ ac->classzone_idx, alloc_flags)) {
-+ int ret;
-+
-+ /* Checked here to keep the fast path fast */
-+ BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
-+ if (alloc_flags & ALLOC_NO_WATERMARKS)
-+ goto try_this_zone;
-+
-+ if (IS_ENABLED(CONFIG_NUMA) &&
-+ !did_zlc_setup && nr_online_nodes > 1) {
-+ /*
-+ * we do zlc_setup if there are multiple nodes
-+ * and before considering the first zone allowed
-+ * by the cpuset.
-+ */
-+ allowednodes = zlc_setup(zonelist, alloc_flags);
-+ zlc_active = 1;
-+ did_zlc_setup = 1;
-+ }
-+
-+ if (zone_reclaim_mode == 0 ||
-+ !zone_allows_reclaim(ac->preferred_zone, zone))
-+ goto this_zone_full;
-+
-+ /*
-+ * As we may have just activated ZLC, check if the first
-+ * eligible zone has failed zone_reclaim recently.
-+ */
-+ if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&
-+ !zlc_zone_worth_trying(zonelist, z, allowednodes))
-+ continue;
-+
-+ ret = zone_reclaim(zone, gfp_mask, order);
-+ switch (ret) {
-+ case ZONE_RECLAIM_NOSCAN:
-+ /* did not scan */
-+ continue;
-+ case ZONE_RECLAIM_FULL:
-+ /* scanned but unreclaimable */
-+ continue;
-+ default:
-+ /* did we reclaim enough */
-+ if (zone_watermark_ok(zone, order, mark,
-+ ac->classzone_idx, alloc_flags))
-+ goto try_this_zone;
-+
-+ /*
-+ * Failed to reclaim enough to meet watermark.
-+ * Only mark the zone full if checking the min
-+ * watermark or if we failed to reclaim just
-+ * 1<<order pages or else the page allocator
-+ * fastpath will prematurely mark zones full
-+ * when the watermark is between the low and
-+ * min watermarks.
-+ */
-+ if (((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN) ||
-+ ret == ZONE_RECLAIM_SOME)
-+ goto this_zone_full;
-+
-+ continue;
-+ }
-+ }
-+
-+try_this_zone:
-+ page = buffered_rmqueue(ac->preferred_zone, zone, order,
-+ gfp_mask, ac->migratetype);
-+ if (page) {
-+ if (prep_new_page(page, order, gfp_mask, alloc_flags))
-+ goto try_this_zone;
-+ return page;
-+ }
-+this_zone_full:
-+ if (IS_ENABLED(CONFIG_NUMA) && zlc_active)
-+ zlc_mark_zone_full(zonelist, z);
-+ }
-+
-+ /*
-+ * The first pass makes sure allocations are spread fairly within the
-+ * local node. However, the local node might have free pages left
-+ * after the fairness batches are exhausted, and remote zones haven't
-+ * even been considered yet. Try once more without fairness, and
-+ * include remote zones now, before entering the slowpath and waking
-+ * kswapd: prefer spilling to a remote zone over swapping locally.
-+ */
-+ if (alloc_flags & ALLOC_FAIR) {
-+ alloc_flags &= ~ALLOC_FAIR;
-+ if (nr_fair_skipped) {
-+ zonelist_rescan = true;
-+ reset_alloc_batches(ac->preferred_zone);
-+ }
-+ if (nr_online_nodes > 1)
-+ zonelist_rescan = true;
-+ }
-+
-+ if (unlikely(IS_ENABLED(CONFIG_NUMA) && zlc_active)) {
-+ /* Disable zlc cache for second zonelist scan */
-+ zlc_active = 0;
-+ zonelist_rescan = true;
-+ }
-+
-+ if (zonelist_rescan)
-+ goto zonelist_scan;
-+
-+ return NULL;
-+}
-+
-+/*
-+ * Large machines with many possible nodes should not always dump per-node
-+ * meminfo in irq context.
-+ */
-+static inline bool should_suppress_show_mem(void)
-+{
-+ bool ret = false;
-+
-+#if NODES_SHIFT > 8
-+ ret = in_interrupt();
-+#endif
-+ return ret;
-+}
-+
-+static DEFINE_RATELIMIT_STATE(nopage_rs,
-+ DEFAULT_RATELIMIT_INTERVAL,
-+ DEFAULT_RATELIMIT_BURST);
-+
-+void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...)
-+{
-+ unsigned int filter = SHOW_MEM_FILTER_NODES;
-+
-+ if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
-+ debug_guardpage_minorder() > 0)
-+ return;
-+
-+ /*
-+ * This documents exceptions given to allocations in certain
-+ * contexts that are allowed to allocate outside current's set
-+ * of allowed nodes.
-+ */
-+ if (!(gfp_mask & __GFP_NOMEMALLOC))
-+ if (test_thread_flag(TIF_MEMDIE) ||
-+ (current->flags & (PF_MEMALLOC | PF_EXITING)))
-+ filter &= ~SHOW_MEM_FILTER_NODES;
-+ if (in_interrupt() || !(gfp_mask & __GFP_WAIT))
-+ filter &= ~SHOW_MEM_FILTER_NODES;
-+
-+ if (fmt) {
-+ struct va_format vaf;
-+ va_list args;
-+
-+ va_start(args, fmt);
-+
-+ vaf.fmt = fmt;
-+ vaf.va = &args;
-+
-+ pr_warn("%pV", &vaf);
-+
-+ va_end(args);
-+ }
-+
-+ pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n",
-+ current->comm, order, gfp_mask);
-+
-+ dump_stack();
-+ if (!should_suppress_show_mem())
-+ show_mem(filter);
-+}
-+
-+static inline int
-+should_alloc_retry(gfp_t gfp_mask, unsigned int order,
-+ unsigned long did_some_progress,
-+ unsigned long pages_reclaimed)
-+{
-+ /* Do not loop if specifically requested */
-+ if (gfp_mask & __GFP_NORETRY)
-+ return 0;
-+
-+ /* Always retry if specifically requested */
-+ if (gfp_mask & __GFP_NOFAIL)
-+ return 1;
-+
-+ /*
-+ * Suspend converts GFP_KERNEL to __GFP_WAIT which can prevent reclaim
-+ * making forward progress without invoking OOM. Suspend also disables
-+ * storage devices so kswapd will not help. Bail if we are suspending.
-+ */
-+ if (!did_some_progress && pm_suspended_storage())
-+ return 0;
-+
-+ /*
-+ * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
-+ * means __GFP_NOFAIL, but that may not be true in other
-+ * implementations.
-+ */
-+ if (order <= PAGE_ALLOC_COSTLY_ORDER)
-+ return 1;
-+
-+ /*
-+ * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
-+ * specified, then we retry until we no longer reclaim any pages
-+ * (above), or we've reclaimed an order of pages at least as
-+ * large as the allocation's order. In both cases, if the
-+ * allocation still fails, we stop retrying.
-+ */
-+ if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
-+ return 1;
-+
-+ return 0;
-+}
-+
-+static inline struct page *
-+__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
-+ const struct alloc_context *ac, unsigned long *did_some_progress)
-+{
-+ struct page *page;
-+
-+ *did_some_progress = 0;
-+
-+ /*
-+ * Acquire the per-zone oom lock for each zone. If that
-+ * fails, somebody else is making progress for us.
-+ */
-+ if (!oom_zonelist_trylock(ac->zonelist, gfp_mask)) {
-+ *did_some_progress = 1;
-+ schedule_timeout_uninterruptible(1);
-+ return NULL;
-+ }
-+
-+ /*
-+ * Go through the zonelist yet one more time, keep very high watermark
-+ * here, this is only to catch a parallel oom killing, we must fail if
-+ * we're still under heavy pressure.
-+ */
-+ page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order,
-+ ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
-+ if (page)
-+ goto out;
-+
-+ if (!(gfp_mask & __GFP_NOFAIL)) {
-+ /* Coredumps can quickly deplete all memory reserves */
-+ if (current->flags & PF_DUMPCORE)
-+ goto out;
-+ /* The OOM killer will not help higher order allocs */
-+ if (order > PAGE_ALLOC_COSTLY_ORDER)
-+ goto out;
-+ /* The OOM killer does not needlessly kill tasks for lowmem */
-+ if (ac->high_zoneidx < ZONE_NORMAL)
-+ goto out;
-+ /* The OOM killer does not compensate for light reclaim */
-+ if (!(gfp_mask & __GFP_FS)) {
-+ /*
-+ * XXX: Page reclaim didn't yield anything,
-+ * and the OOM killer can't be invoked, but
-+ * keep looping as per should_alloc_retry().
-+ */
-+ *did_some_progress = 1;
-+ goto out;
-+ }
-+ /* The OOM killer may not free memory on a specific node */
-+ if (gfp_mask & __GFP_THISNODE)
-+ goto out;
-+ }
-+ /* Exhausted what can be done so it's blamo time */
-+ if (out_of_memory(ac->zonelist, gfp_mask, order, ac->nodemask, false)
-+ || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL))
-+ *did_some_progress = 1;
-+out:
-+ oom_zonelist_unlock(ac->zonelist, gfp_mask);
-+ return page;
-+}
-+
-+#ifdef CONFIG_COMPACTION
-+/* Try memory compaction for high-order allocations before reclaim */
-+static struct page *
-+__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
-+ int alloc_flags, const struct alloc_context *ac,
-+ enum migrate_mode mode, int *contended_compaction,
-+ bool *deferred_compaction)
-+{
-+ unsigned long compact_result;
-+ struct page *page;
-+
-+ if (!order)
-+ return NULL;
-+
-+ current->flags |= PF_MEMALLOC;
-+ compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
-+ mode, contended_compaction);
-+ current->flags &= ~PF_MEMALLOC;
-+
-+ switch (compact_result) {
-+ case COMPACT_DEFERRED:
-+ *deferred_compaction = true;
-+ /* fall-through */
-+ case COMPACT_SKIPPED:
-+ return NULL;
-+ default:
-+ break;
-+ }
-+
-+ /*
-+ * At least in one zone compaction wasn't deferred or skipped, so let's
-+ * count a compaction stall
-+ */
-+ count_vm_event(COMPACTSTALL);
-+
-+ page = get_page_from_freelist(gfp_mask, order,
-+ alloc_flags & ~ALLOC_NO_WATERMARKS, ac);
-+
-+ if (page) {
-+ struct zone *zone = page_zone(page);
-+
-+ zone->compact_blockskip_flush = false;
-+ compaction_defer_reset(zone, order, true);
-+ count_vm_event(COMPACTSUCCESS);
-+ return page;
-+ }
-+
-+ /*
-+ * It's bad if compaction run occurs and fails. The most likely reason
-+ * is that pages exist, but not enough to satisfy watermarks.
-+ */
-+ count_vm_event(COMPACTFAIL);
-+
-+ cond_resched();
-+
-+ return NULL;
-+}
-+#else
-+static inline struct page *
-+__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
-+ int alloc_flags, const struct alloc_context *ac,
-+ enum migrate_mode mode, int *contended_compaction,
-+ bool *deferred_compaction)
-+{
-+ return NULL;
-+}
-+#endif /* CONFIG_COMPACTION */
-+
-+/* Perform direct synchronous page reclaim */
-+static int
-+__perform_reclaim(gfp_t gfp_mask, unsigned int order,
-+ const struct alloc_context *ac)
-+{
-+ struct reclaim_state reclaim_state;
-+ int progress;
-+
-+ cond_resched();
-+
-+ /* We now go into synchronous reclaim */
-+ cpuset_memory_pressure_bump();
-+ current->flags |= PF_MEMALLOC;
-+ lockdep_set_current_reclaim_state(gfp_mask);
-+ reclaim_state.reclaimed_slab = 0;
-+ current->reclaim_state = &reclaim_state;
-+
-+ progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
-+ ac->nodemask);
-+
-+ current->reclaim_state = NULL;
-+ lockdep_clear_current_reclaim_state();
-+ current->flags &= ~PF_MEMALLOC;
-+
-+ cond_resched();
-+
-+ return progress;
-+}
-+
-+/* The really slow allocator path where we enter direct reclaim */
-+static inline struct page *
-+__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
-+ int alloc_flags, const struct alloc_context *ac,
-+ unsigned long *did_some_progress)
-+{
-+ struct page *page = NULL;
-+ bool drained = false;
-+
-+ *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
-+ if (unlikely(!(*did_some_progress)))
-+ return NULL;
-+
-+ /* After successful reclaim, reconsider all zones for allocation */
-+ if (IS_ENABLED(CONFIG_NUMA))
-+ zlc_clear_zones_full(ac->zonelist);
-+
-+retry:
-+ page = get_page_from_freelist(gfp_mask, order,
-+ alloc_flags & ~ALLOC_NO_WATERMARKS, ac);
-+
-+ /*
-+ * If an allocation failed after direct reclaim, it could be because
-+ * pages are pinned on the per-cpu lists. Drain them and try again
-+ */
-+ if (!page && !drained) {
-+ drain_all_pages(NULL);
-+ drained = true;
-+ goto retry;
-+ }
-+
-+ return page;
-+}
-+
-+/*
-+ * This is called in the allocator slow-path if the allocation request is of
-+ * sufficient urgency to ignore watermarks and take other desperate measures
-+ */
-+static inline struct page *
-+__alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
-+ const struct alloc_context *ac)
-+{
-+ struct page *page;
-+
-+ do {
-+ page = get_page_from_freelist(gfp_mask, order,
-+ ALLOC_NO_WATERMARKS, ac);
-+
-+ if (!page && gfp_mask & __GFP_NOFAIL)
-+ wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC,
-+ HZ/50);
-+ } while (!page && (gfp_mask & __GFP_NOFAIL));
-+
-+ return page;
-+}
-+
-+static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac)
-+{
-+ struct zoneref *z;
-+ struct zone *zone;
-+
-+ for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
-+ ac->high_zoneidx, ac->nodemask)
-+ wakeup_kswapd(zone, order, zone_idx(ac->preferred_zone));
-+}
-+
-+static inline int
-+gfp_to_alloc_flags(gfp_t gfp_mask)
-+{
-+ int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
-+ const bool atomic = !(gfp_mask & (__GFP_WAIT | __GFP_NO_KSWAPD));
-+
-+ /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
-+ BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
-+
-+ /*
-+ * The caller may dip into page reserves a bit more if the caller
-+ * cannot run direct reclaim, or if the caller has realtime scheduling
-+ * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
-+ * set both ALLOC_HARDER (atomic == true) and ALLOC_HIGH (__GFP_HIGH).
-+ */
-+ alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
-+
-+ if (atomic) {
-+ /*
-+ * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
-+ * if it can't schedule.
-+ */
-+ if (!(gfp_mask & __GFP_NOMEMALLOC))
-+ alloc_flags |= ALLOC_HARDER;
-+ /*
-+ * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
-+ * comment for __cpuset_node_allowed().
-+ */
-+ alloc_flags &= ~ALLOC_CPUSET;
-+ } else if (unlikely(rt_task(current)) && !in_interrupt())
-+ alloc_flags |= ALLOC_HARDER;
-+
-+ if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
-+ if (gfp_mask & __GFP_MEMALLOC)
-+ alloc_flags |= ALLOC_NO_WATERMARKS;
-+ else if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
-+ alloc_flags |= ALLOC_NO_WATERMARKS;
-+ else if (!in_interrupt() &&
-+ ((current->flags & PF_MEMALLOC) ||
-+ unlikely(test_thread_flag(TIF_MEMDIE))))
-+ alloc_flags |= ALLOC_NO_WATERMARKS;
-+ }
-+#ifdef CONFIG_CMA
-+ if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
-+ alloc_flags |= ALLOC_CMA;
-+#endif
-+ return alloc_flags;
-+}
-+
-+bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
-+{
-+ return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS);
-+}
-+
-+static inline struct page *
-+__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
-+ struct alloc_context *ac)
-+{
-+ const gfp_t wait = gfp_mask & __GFP_WAIT;
-+ struct page *page = NULL;
-+ int alloc_flags;
-+ unsigned long pages_reclaimed = 0;
-+ unsigned long did_some_progress;
-+ enum migrate_mode migration_mode = MIGRATE_ASYNC;
-+ bool deferred_compaction = false;
-+ int contended_compaction = COMPACT_CONTENDED_NONE;
-+
-+ /*
-+ * In the slowpath, we sanity check order to avoid ever trying to
-+ * reclaim >= MAX_ORDER areas which will never succeed. Callers may
-+ * be using allocators in order of preference for an area that is
-+ * too large.
-+ */
-+ if (order >= MAX_ORDER) {
-+ WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
-+ return NULL;
-+ }
-+
-+ /*
-+ * If this allocation cannot block and it is for a specific node, then
-+ * fail early. There's no need to wakeup kswapd or retry for a
-+ * speculative node-specific allocation.
-+ */
-+ if (IS_ENABLED(CONFIG_NUMA) && (gfp_mask & __GFP_THISNODE) && !wait)
-+ goto nopage;
-+
-+retry:
-+ if (!(gfp_mask & __GFP_NO_KSWAPD))
-+ wake_all_kswapds(order, ac);
-+
-+ /*
-+ * OK, we're below the kswapd watermark and have kicked background
-+ * reclaim. Now things get more complex, so set up alloc_flags according
-+ * to how we want to proceed.
-+ */
-+ alloc_flags = gfp_to_alloc_flags(gfp_mask);
-+
-+ /*
-+ * Find the true preferred zone if the allocation is unconstrained by
-+ * cpusets.
-+ */
-+ if (!(alloc_flags & ALLOC_CPUSET) && !ac->nodemask) {
-+ struct zoneref *preferred_zoneref;
-+ preferred_zoneref = first_zones_zonelist(ac->zonelist,
-+ ac->high_zoneidx, NULL, &ac->preferred_zone);
-+ ac->classzone_idx = zonelist_zone_idx(preferred_zoneref);
-+ }
-+
-+ /* This is the last chance, in general, before the goto nopage. */
-+ page = get_page_from_freelist(gfp_mask, order,
-+ alloc_flags & ~ALLOC_NO_WATERMARKS, ac);
-+ if (page)
-+ goto got_pg;
-+
-+ /* Allocate without watermarks if the context allows */
-+ if (alloc_flags & ALLOC_NO_WATERMARKS) {
-+ /*
-+ * Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds
-+ * the allocation is high priority and these type of
-+ * allocations are system rather than user orientated
-+ */
-+ ac->zonelist = node_zonelist(numa_node_id(), gfp_mask);
-+
-+ page = __alloc_pages_high_priority(gfp_mask, order, ac);
-+
-+ if (page) {
-+ goto got_pg;
-+ }
-+ }
-+
-+ /* Atomic allocations - we can't balance anything */
-+ if (!wait) {
-+ /*
-+ * All existing users of the deprecated __GFP_NOFAIL are
-+ * blockable, so warn of any new users that actually allow this
-+ * type of allocation to fail.
-+ */
-+ WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL);
-+ goto nopage;
-+ }
-+
-+ /* Avoid recursion of direct reclaim */
-+ if (current->flags & PF_MEMALLOC)
-+ goto nopage;
-+
-+ /* Avoid allocations with no watermarks from looping endlessly */
-+ if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
-+ goto nopage;
-+
-+ /*
-+ * Try direct compaction. The first pass is asynchronous. Subsequent
-+ * attempts after direct reclaim are synchronous
-+ */
-+ page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
-+ migration_mode,
-+ &contended_compaction,
-+ &deferred_compaction);
-+ if (page)
-+ goto got_pg;
-+
-+ /* Checks for THP-specific high-order allocations */
-+ if ((gfp_mask & GFP_TRANSHUGE) == GFP_TRANSHUGE) {
-+ /*
-+ * If compaction is deferred for high-order allocations, it is
-+ * because sync compaction recently failed. If this is the case
-+ * and the caller requested a THP allocation, we do not want
-+ * to heavily disrupt the system, so we fail the allocation
-+ * instead of entering direct reclaim.
-+ */
-+ if (deferred_compaction)
-+ goto nopage;
-+
-+ /*
-+ * In all zones where compaction was attempted (and not
-+ * deferred or skipped), lock contention has been detected.
-+ * For THP allocation we do not want to disrupt the others
-+ * so we fallback to base pages instead.
-+ */
-+ if (contended_compaction == COMPACT_CONTENDED_LOCK)
-+ goto nopage;
-+
-+ /*
-+ * If compaction was aborted due to need_resched(), we do not
-+ * want to further increase allocation latency, unless it is
-+ * khugepaged trying to collapse.
-+ */
-+ if (contended_compaction == COMPACT_CONTENDED_SCHED
-+ && !(current->flags & PF_KTHREAD))
-+ goto nopage;
-+ }
-+
-+ /*
-+ * It can become very expensive to allocate transparent hugepages at
-+ * fault, so use asynchronous memory compaction for THP unless it is
-+ * khugepaged trying to collapse.
-+ */
-+ if ((gfp_mask & GFP_TRANSHUGE) != GFP_TRANSHUGE ||
-+ (current->flags & PF_KTHREAD))
-+ migration_mode = MIGRATE_SYNC_LIGHT;
-+
-+ /* Try direct reclaim and then allocating */
-+ page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
-+ &did_some_progress);
-+ if (page)
-+ goto got_pg;
-+
-+ /* Check if we should retry the allocation */
-+ pages_reclaimed += did_some_progress;
-+ if (should_alloc_retry(gfp_mask, order, did_some_progress,
-+ pages_reclaimed)) {
-+ /*
-+ * If we fail to make progress by freeing individual
-+ * pages, but the allocation wants us to keep going,
-+ * start OOM killing tasks.
-+ */
-+ if (!did_some_progress) {
-+ page = __alloc_pages_may_oom(gfp_mask, order, ac,
-+ &did_some_progress);
-+ if (page)
-+ goto got_pg;
-+ if (!did_some_progress)
-+ goto nopage;
-+ }
-+ /* Wait for some write requests to complete then retry */
-+ wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, HZ/50);
-+ goto retry;
-+ } else {
-+ /*
-+ * High-order allocations do not necessarily loop after
-+ * direct reclaim and reclaim/compaction depends on compaction
-+ * being called after reclaim so call directly if necessary
-+ */
-+ page = __alloc_pages_direct_compact(gfp_mask, order,
-+ alloc_flags, ac, migration_mode,
-+ &contended_compaction,
-+ &deferred_compaction);
-+ if (page)
-+ goto got_pg;
-+ }
-+
-+nopage:
-+ warn_alloc_failed(gfp_mask, order, NULL);
-+got_pg:
-+ return page;
-+}
-+
-+/*
-+ * This is the 'heart' of the zoned buddy allocator.
-+ */
-+struct page *
-+__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
-+ struct zonelist *zonelist, nodemask_t *nodemask)
-+{
-+ struct zoneref *preferred_zoneref;
-+ struct page *page = NULL;
-+ unsigned int cpuset_mems_cookie;
-+ int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR;
-+ gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
-+ struct alloc_context ac = {
-+ .high_zoneidx = gfp_zone(gfp_mask),
-+ .nodemask = nodemask,
-+ .migratetype = gfpflags_to_migratetype(gfp_mask),
-+ };
-+
-+ gfp_mask &= gfp_allowed_mask;
-+
-+ lockdep_trace_alloc(gfp_mask);
-+
-+ might_sleep_if(gfp_mask & __GFP_WAIT);
-+
-+ if (should_fail_alloc_page(gfp_mask, order))
-+ return NULL;
-+
-+ /*
-+ * Check the zones suitable for the gfp_mask contain at least one
-+ * valid zone. It's possible to have an empty zonelist as a result
-+ * of __GFP_THISNODE and a memoryless node
-+ */
-+ if (unlikely(!zonelist->_zonerefs->zone))
-+ return NULL;
-+
-+ if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE)
-+ alloc_flags |= ALLOC_CMA;
-+
-+retry_cpuset:
-+ cpuset_mems_cookie = read_mems_allowed_begin();
-+
-+ /* We set it here, as __alloc_pages_slowpath might have changed it */
-+ ac.zonelist = zonelist;
-+ /* The preferred zone is used for statistics later */
-+ preferred_zoneref = first_zones_zonelist(ac.zonelist, ac.high_zoneidx,
-+ ac.nodemask ? : &cpuset_current_mems_allowed,
-+ &ac.preferred_zone);
-+ if (!ac.preferred_zone)
-+ goto out;
-+ ac.classzone_idx = zonelist_zone_idx(preferred_zoneref);
-+
-+ /* First allocation attempt */
-+ alloc_mask = gfp_mask|__GFP_HARDWALL;
-+ page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
-+ if (unlikely(!page)) {
-+ /*
-+ * Runtime PM, block IO and its error handling path
-+ * can deadlock because I/O on the device might not
-+ * complete.
-+ */
-+ alloc_mask = memalloc_noio_flags(gfp_mask);
-+
-+ page = __alloc_pages_slowpath(alloc_mask, order, &ac);
-+ }
-+
-+ if (kmemcheck_enabled && page)
-+ kmemcheck_pagealloc_alloc(page, order, gfp_mask);
-+
-+ trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
-+
-+out:
-+ /*
-+ * When updating a task's mems_allowed, it is possible to race with
-+ * parallel threads in such a way that an allocation can fail while
-+ * the mask is being updated. If a page allocation is about to fail,
-+ * check if the cpuset changed during allocation and if so, retry.
-+ */
-+ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
-+ goto retry_cpuset;
-+
-+ return page;
-+}
-+EXPORT_SYMBOL(__alloc_pages_nodemask);
-+
-+/*
-+ * Common helper functions.
-+ */
-+unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
-+{
-+ struct page *page;
-+
-+ /*
-+ * __get_free_pages() returns a 32-bit address, which cannot represent
-+ * a highmem page
-+ */
-+ VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
-+
-+ page = alloc_pages(gfp_mask, order);
-+ if (!page)
-+ return 0;
-+ return (unsigned long) page_address(page);
-+}
-+EXPORT_SYMBOL(__get_free_pages);
-+
-+unsigned long get_zeroed_page(gfp_t gfp_mask)
-+{
-+ return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
-+}
-+EXPORT_SYMBOL(get_zeroed_page);
-+
-+void __free_pages(struct page *page, unsigned int order)
-+{
-+ if (put_page_testzero(page)) {
-+ if (order == 0)
-+ free_hot_cold_page(page, false);
-+ else
-+ __free_pages_ok(page, order);
-+ }
-+}
-+
-+EXPORT_SYMBOL(__free_pages);
-+
-+void free_pages(unsigned long addr, unsigned int order)
-+{
-+ if (addr != 0) {
-+ VM_BUG_ON(!virt_addr_valid((void *)addr));
-+ __free_pages(virt_to_page((void *)addr), order);
-+ }
-+}
-+
-+EXPORT_SYMBOL(free_pages);
-+
-+/*
-+ * alloc_kmem_pages charges newly allocated pages to the kmem resource counter
-+ * of the current memory cgroup.
-+ *
-+ * It should be used when the caller would like to use kmalloc, but since the
-+ * allocation is large, it has to fall back to the page allocator.
-+ */
-+struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order)
-+{
-+ struct page *page;
-+ struct mem_cgroup *memcg = NULL;
-+
-+ if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order))
-+ return NULL;
-+ page = alloc_pages(gfp_mask, order);
-+ memcg_kmem_commit_charge(page, memcg, order);
-+ return page;
-+}
-+
-+struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask, unsigned int order)
-+{
-+ struct page *page;
-+ struct mem_cgroup *memcg = NULL;
-+
-+ if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order))
-+ return NULL;
-+ page = alloc_pages_node(nid, gfp_mask, order);
-+ memcg_kmem_commit_charge(page, memcg, order);
-+ return page;
-+}
-+
-+/*
-+ * __free_kmem_pages and free_kmem_pages will free pages allocated with
-+ * alloc_kmem_pages.
-+ */
-+void __free_kmem_pages(struct page *page, unsigned int order)
-+{
-+ memcg_kmem_uncharge_pages(page, order);
-+ __free_pages(page, order);
-+}
-+
-+void free_kmem_pages(unsigned long addr, unsigned int order)
-+{
-+ if (addr != 0) {
-+ VM_BUG_ON(!virt_addr_valid((void *)addr));
-+ __free_kmem_pages(virt_to_page((void *)addr), order);
-+ }
-+}
-+
-+static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size)
-+{
-+ if (addr) {
-+ unsigned long alloc_end = addr + (PAGE_SIZE << order);
-+ unsigned long used = addr + PAGE_ALIGN(size);
-+
-+ split_page(virt_to_page((void *)addr), order);
-+ while (used < alloc_end) {
-+ free_page(used);
-+ used += PAGE_SIZE;
-+ }
-+ }
-+ return (void *)addr;
-+}
-+
-+/**
-+ * alloc_pages_exact - allocate an exact number physically-contiguous pages.
-+ * @size: the number of bytes to allocate
-+ * @gfp_mask: GFP flags for the allocation
-+ *
-+ * This function is similar to alloc_pages(), except that it allocates the
-+ * minimum number of pages to satisfy the request. alloc_pages() can only
-+ * allocate memory in power-of-two pages.
-+ *
-+ * This function is also limited by MAX_ORDER.
-+ *
-+ * Memory allocated by this function must be released by free_pages_exact().
-+ */
-+void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
-+{
-+ unsigned int order = get_order(size);
-+ unsigned long addr;
-+
-+ addr = __get_free_pages(gfp_mask, order);
-+ return make_alloc_exact(addr, order, size);
-+}
-+EXPORT_SYMBOL(alloc_pages_exact);
-+
-+/**
-+ * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
-+ * pages on a node.
-+ * @nid: the preferred node ID where memory should be allocated
-+ * @size: the number of bytes to allocate
-+ * @gfp_mask: GFP flags for the allocation
-+ *
-+ * Like alloc_pages_exact(), but try to allocate on node nid first before falling
-+ * back.
-+ * Note this is not alloc_pages_exact_node() which allocates on a specific node,
-+ * but is not exact.
-+ */
-+void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
-+{
-+ unsigned order = get_order(size);
-+ struct page *p = alloc_pages_node(nid, gfp_mask, order);
-+ if (!p)
-+ return NULL;
-+ return make_alloc_exact((unsigned long)page_address(p), order, size);
-+}
-+
-+/**
-+ * free_pages_exact - release memory allocated via alloc_pages_exact()
-+ * @virt: the value returned by alloc_pages_exact.
-+ * @size: size of allocation, same value as passed to alloc_pages_exact().
-+ *
-+ * Release the memory allocated by a previous call to alloc_pages_exact.
-+ */
-+void free_pages_exact(void *virt, size_t size)
-+{
-+ unsigned long addr = (unsigned long)virt;
-+ unsigned long end = addr + PAGE_ALIGN(size);
-+
-+ while (addr < end) {
-+ free_page(addr);
-+ addr += PAGE_SIZE;
-+ }
-+}
-+EXPORT_SYMBOL(free_pages_exact);
-+
-+/**
-+ * nr_free_zone_pages - count number of pages beyond high watermark
-+ * @offset: The zone index of the highest zone
-+ *
-+ * nr_free_zone_pages() counts the number of counts pages which are beyond the
-+ * high watermark within all zones at or below a given zone index. For each
-+ * zone, the number of pages is calculated as:
-+ * managed_pages - high_pages
-+ */
-+static unsigned long nr_free_zone_pages(int offset)
-+{
-+ struct zoneref *z;
-+ struct zone *zone;
-+
-+ /* Just pick one node, since fallback list is circular */
-+ unsigned long sum = 0;
-+
-+ struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
-+
-+ for_each_zone_zonelist(zone, z, zonelist, offset) {
-+ unsigned long size = zone->managed_pages;
-+ unsigned long high = high_wmark_pages(zone);
-+ if (size > high)
-+ sum += size - high;
-+ }
-+
-+ return sum;
-+}
-+
-+/**
-+ * nr_free_buffer_pages - count number of pages beyond high watermark
-+ *
-+ * nr_free_buffer_pages() counts the number of pages which are beyond the high
-+ * watermark within ZONE_DMA and ZONE_NORMAL.
-+ */
-+unsigned long nr_free_buffer_pages(void)
-+{
-+ return nr_free_zone_pages(gfp_zone(GFP_USER));
-+}
-+EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
-+
-+/**
-+ * nr_free_pagecache_pages - count number of pages beyond high watermark
-+ *
-+ * nr_free_pagecache_pages() counts the number of pages which are beyond the
-+ * high watermark within all zones.
-+ */
-+unsigned long nr_free_pagecache_pages(void)
-+{
-+ return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
-+}
-+
-+static inline void show_node(struct zone *zone)
-+{
-+ if (IS_ENABLED(CONFIG_NUMA))
-+ printk("Node %d ", zone_to_nid(zone));
-+}
-+
-+void si_meminfo(struct sysinfo *val)
-+{
-+ val->totalram = totalram_pages;
-+ val->sharedram = global_page_state(NR_SHMEM);
-+ val->freeram = global_page_state(NR_FREE_PAGES);
-+ val->bufferram = nr_blockdev_pages();
-+ val->totalhigh = totalhigh_pages;
-+ val->freehigh = nr_free_highpages();
-+ val->mem_unit = PAGE_SIZE;
-+}
-+
-+EXPORT_SYMBOL(si_meminfo);
-+
-+#ifdef CONFIG_NUMA
-+void si_meminfo_node(struct sysinfo *val, int nid)
-+{
-+ int zone_type; /* needs to be signed */
-+ unsigned long managed_pages = 0;
-+ pg_data_t *pgdat = NODE_DATA(nid);
-+
-+ for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
-+ managed_pages += pgdat->node_zones[zone_type].managed_pages;
-+ val->totalram = managed_pages;
-+ val->sharedram = node_page_state(nid, NR_SHMEM);
-+ val->freeram = node_page_state(nid, NR_FREE_PAGES);
-+#ifdef CONFIG_HIGHMEM
-+ val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].managed_pages;
-+ val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
-+ NR_FREE_PAGES);
-+#else
-+ val->totalhigh = 0;
-+ val->freehigh = 0;
-+#endif
-+ val->mem_unit = PAGE_SIZE;
-+}
-+#endif
-+
-+/*
-+ * Determine whether the node should be displayed or not, depending on whether
-+ * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
-+ */
-+bool skip_free_areas_node(unsigned int flags, int nid)
-+{
-+ bool ret = false;
-+ unsigned int cpuset_mems_cookie;
-+
-+ if (!(flags & SHOW_MEM_FILTER_NODES))
-+ goto out;
-+
-+ do {
-+ cpuset_mems_cookie = read_mems_allowed_begin();
-+ ret = !node_isset(nid, cpuset_current_mems_allowed);
-+ } while (read_mems_allowed_retry(cpuset_mems_cookie));
-+out:
-+ return ret;
-+}
-+
-+#define K(x) ((x) << (PAGE_SHIFT-10))
-+
-+static void show_migration_types(unsigned char type)
-+{
-+ static const char types[MIGRATE_TYPES] = {
-+ [MIGRATE_UNMOVABLE] = 'U',
-+ [MIGRATE_RECLAIMABLE] = 'E',
-+ [MIGRATE_MOVABLE] = 'M',
-+ [MIGRATE_RESERVE] = 'R',
-+#ifdef CONFIG_CMA
-+ [MIGRATE_CMA] = 'C',
-+#endif
-+#ifdef CONFIG_MEMORY_ISOLATION
-+ [MIGRATE_ISOLATE] = 'I',
-+#endif
-+ };
-+ char tmp[MIGRATE_TYPES + 1];
-+ char *p = tmp;
-+ int i;
-+
-+ for (i = 0; i < MIGRATE_TYPES; i++) {
-+ if (type & (1 << i))
-+ *p++ = types[i];
-+ }
-+
-+ *p = '\0';
-+ printk("(%s) ", tmp);
-+}
-+
-+/*
-+ * Show free area list (used inside shift_scroll-lock stuff)
-+ * We also calculate the percentage fragmentation. We do this by counting the
-+ * memory on each free list with the exception of the first item on the list.
-+ *
-+ * Bits in @filter:
-+ * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
-+ * cpuset.
-+ */
-+void show_free_areas(unsigned int filter)
-+{
-+ unsigned long free_pcp = 0;
-+ int cpu;
-+ struct zone *zone;
-+
-+ for_each_populated_zone(zone) {
-+ if (skip_free_areas_node(filter, zone_to_nid(zone)))
-+ continue;
-+
-+ for_each_online_cpu(cpu)
-+ free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
-+ }
-+
-+ printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
-+ " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
-+ " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
-+ " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
-+ " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
-+ " free:%lu free_pcp:%lu free_cma:%lu\n",
-+ global_page_state(NR_ACTIVE_ANON),
-+ global_page_state(NR_INACTIVE_ANON),
-+ global_page_state(NR_ISOLATED_ANON),
-+ global_page_state(NR_ACTIVE_FILE),
-+ global_page_state(NR_INACTIVE_FILE),
-+ global_page_state(NR_ISOLATED_FILE),
-+ global_page_state(NR_UNEVICTABLE),
-+ global_page_state(NR_FILE_DIRTY),
-+ global_page_state(NR_WRITEBACK),
-+ global_page_state(NR_UNSTABLE_NFS),
-+ global_page_state(NR_SLAB_RECLAIMABLE),
-+ global_page_state(NR_SLAB_UNRECLAIMABLE),
-+ global_page_state(NR_FILE_MAPPED),
-+ global_page_state(NR_SHMEM),
-+ global_page_state(NR_PAGETABLE),
-+ global_page_state(NR_BOUNCE),
-+ global_page_state(NR_FREE_PAGES),
-+ free_pcp,
-+ global_page_state(NR_FREE_CMA_PAGES));
-+
-+ for_each_populated_zone(zone) {
-+ int i;
-+
-+ if (skip_free_areas_node(filter, zone_to_nid(zone)))
-+ continue;
-+
-+ free_pcp = 0;
-+ for_each_online_cpu(cpu)
-+ free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
-+
-+ show_node(zone);
-+ printk("%s"
-+ " free:%lukB"
-+ " min:%lukB"
-+ " low:%lukB"
-+ " high:%lukB"
-+ " active_anon:%lukB"
-+ " inactive_anon:%lukB"
-+ " active_file:%lukB"
-+ " inactive_file:%lukB"
-+ " unevictable:%lukB"
-+ " isolated(anon):%lukB"
-+ " isolated(file):%lukB"
-+ " present:%lukB"
-+ " managed:%lukB"
-+ " mlocked:%lukB"
-+ " dirty:%lukB"
-+ " writeback:%lukB"
-+ " mapped:%lukB"
-+ " shmem:%lukB"
-+ " slab_reclaimable:%lukB"
-+ " slab_unreclaimable:%lukB"
-+ " kernel_stack:%lukB"
-+ " pagetables:%lukB"
-+ " unstable:%lukB"
-+ " bounce:%lukB"
-+ " free_pcp:%lukB"
-+ " local_pcp:%ukB"
-+ " free_cma:%lukB"
-+ " writeback_tmp:%lukB"
-+ " pages_scanned:%lu"
-+ " all_unreclaimable? %s"
-+ "\n",
-+ zone->name,
-+ K(zone_page_state(zone, NR_FREE_PAGES)),
-+ K(min_wmark_pages(zone)),
-+ K(low_wmark_pages(zone)),
-+ K(high_wmark_pages(zone)),
-+ K(zone_page_state(zone, NR_ACTIVE_ANON)),
-+ K(zone_page_state(zone, NR_INACTIVE_ANON)),
-+ K(zone_page_state(zone, NR_ACTIVE_FILE)),
-+ K(zone_page_state(zone, NR_INACTIVE_FILE)),
-+ K(zone_page_state(zone, NR_UNEVICTABLE)),
-+ K(zone_page_state(zone, NR_ISOLATED_ANON)),
-+ K(zone_page_state(zone, NR_ISOLATED_FILE)),
-+ K(zone->present_pages),
-+ K(zone->managed_pages),
-+ K(zone_page_state(zone, NR_MLOCK)),
-+ K(zone_page_state(zone, NR_FILE_DIRTY)),
-+ K(zone_page_state(zone, NR_WRITEBACK)),
-+ K(zone_page_state(zone, NR_FILE_MAPPED)),
-+ K(zone_page_state(zone, NR_SHMEM)),
-+ K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
-+ K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
-+ zone_page_state(zone, NR_KERNEL_STACK) *
-+ THREAD_SIZE / 1024,
-+ K(zone_page_state(zone, NR_PAGETABLE)),
-+ K(zone_page_state(zone, NR_UNSTABLE_NFS)),
-+ K(zone_page_state(zone, NR_BOUNCE)),
-+ K(free_pcp),
-+ K(this_cpu_read(zone->pageset->pcp.count)),
-+ K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
-+ K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
-+ K(zone_page_state(zone, NR_PAGES_SCANNED)),
-+ (!zone_reclaimable(zone) ? "yes" : "no")
-+ );
-+ printk("lowmem_reserve[]:");
-+ for (i = 0; i < MAX_NR_ZONES; i++)
-+ printk(" %ld", zone->lowmem_reserve[i]);
-+ printk("\n");
-+ }
-+
-+ for_each_populated_zone(zone) {
-+ unsigned long nr[MAX_ORDER], flags, order, total = 0;
-+ unsigned char types[MAX_ORDER];
-+
-+ if (skip_free_areas_node(filter, zone_to_nid(zone)))
-+ continue;
-+ show_node(zone);
-+ printk("%s: ", zone->name);
-+
-+ spin_lock_irqsave(&zone->lock, flags);
-+ for (order = 0; order < MAX_ORDER; order++) {
-+ struct free_area *area = &zone->free_area[order];
-+ int type;
-+
-+ nr[order] = area->nr_free;
-+ total += nr[order] << order;
-+
-+ types[order] = 0;
-+ for (type = 0; type < MIGRATE_TYPES; type++) {
-+ if (!list_empty(&area->free_list[type]))
-+ types[order] |= 1 << type;
-+ }
-+ }
-+ spin_unlock_irqrestore(&zone->lock, flags);
-+ for (order = 0; order < MAX_ORDER; order++) {
-+ printk("%lu*%lukB ", nr[order], K(1UL) << order);
-+ if (nr[order])
-+ show_migration_types(types[order]);
-+ }
-+ printk("= %lukB\n", K(total));
-+ }
-+
-+ hugetlb_show_meminfo();
-+
-+ printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
-+
-+ show_swap_cache_info();
-+}
-+
-+static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
-+{
-+ zoneref->zone = zone;
-+ zoneref->zone_idx = zone_idx(zone);
-+}
-+
-+/*
-+ * Builds allocation fallback zone lists.
-+ *
-+ * Add all populated zones of a node to the zonelist.
-+ */
-+static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
-+ int nr_zones)
-+{
-+ struct zone *zone;
-+ enum zone_type zone_type = MAX_NR_ZONES;
-+
-+ do {
-+ zone_type--;
-+ zone = pgdat->node_zones + zone_type;
-+ if (populated_zone(zone)) {
-+ zoneref_set_zone(zone,
-+ &zonelist->_zonerefs[nr_zones++]);
-+ check_highest_zone(zone_type);
-+ }
-+ } while (zone_type);
-+
-+ return nr_zones;
-+}
-+
-+
-+/*
-+ * zonelist_order:
-+ * 0 = automatic detection of better ordering.
-+ * 1 = order by ([node] distance, -zonetype)
-+ * 2 = order by (-zonetype, [node] distance)
-+ *
-+ * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
-+ * the same zonelist. So only NUMA can configure this param.
-+ */
-+#define ZONELIST_ORDER_DEFAULT 0
-+#define ZONELIST_ORDER_NODE 1
-+#define ZONELIST_ORDER_ZONE 2
-+
-+/* zonelist order in the kernel.
-+ * set_zonelist_order() will set this to NODE or ZONE.
-+ */
-+static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
-+static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
-+
-+
-+#ifdef CONFIG_NUMA
-+/* The value user specified ....changed by config */
-+static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
-+/* string for sysctl */
-+#define NUMA_ZONELIST_ORDER_LEN 16
-+char numa_zonelist_order[16] = "default";
-+
-+/*
-+ * interface for configure zonelist ordering.
-+ * command line option "numa_zonelist_order"
-+ * = "[dD]efault - default, automatic configuration.
-+ * = "[nN]ode - order by node locality, then by zone within node
-+ * = "[zZ]one - order by zone, then by locality within zone
-+ */
-+
-+static int __parse_numa_zonelist_order(char *s)
-+{
-+ if (*s == 'd' || *s == 'D') {
-+ user_zonelist_order = ZONELIST_ORDER_DEFAULT;
-+ } else if (*s == 'n' || *s == 'N') {
-+ user_zonelist_order = ZONELIST_ORDER_NODE;
-+ } else if (*s == 'z' || *s == 'Z') {
-+ user_zonelist_order = ZONELIST_ORDER_ZONE;
-+ } else {
-+ printk(KERN_WARNING
-+ "Ignoring invalid numa_zonelist_order value: "
-+ "%s\n", s);
-+ return -EINVAL;
-+ }
-+ return 0;
-+}
-+
-+static __init int setup_numa_zonelist_order(char *s)
-+{
-+ int ret;
-+
-+ if (!s)
-+ return 0;
-+
-+ ret = __parse_numa_zonelist_order(s);
-+ if (ret == 0)
-+ strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
-+
-+ return ret;
-+}
-+early_param("numa_zonelist_order", setup_numa_zonelist_order);
-+
-+/*
-+ * sysctl handler for numa_zonelist_order
-+ */
-+int numa_zonelist_order_handler(struct ctl_table *table, int write,
-+ void __user *buffer, size_t *length,
-+ loff_t *ppos)
-+{
-+ char saved_string[NUMA_ZONELIST_ORDER_LEN];
-+ int ret;
-+ static DEFINE_MUTEX(zl_order_mutex);
-+
-+ mutex_lock(&zl_order_mutex);
-+ if (write) {
-+ if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) {
-+ ret = -EINVAL;
-+ goto out;
-+ }
-+ strcpy(saved_string, (char *)table->data);
-+ }
-+ ret = proc_dostring(table, write, buffer, length, ppos);
-+ if (ret)
-+ goto out;
-+ if (write) {
-+ int oldval = user_zonelist_order;
-+
-+ ret = __parse_numa_zonelist_order((char *)table->data);
-+ if (ret) {
-+ /*
-+ * bogus value. restore saved string
-+ */
-+ strncpy((char *)table->data, saved_string,
-+ NUMA_ZONELIST_ORDER_LEN);
-+ user_zonelist_order = oldval;
-+ } else if (oldval != user_zonelist_order) {
-+ mutex_lock(&zonelists_mutex);
-+ build_all_zonelists(NULL, NULL);
-+ mutex_unlock(&zonelists_mutex);
-+ }
-+ }
-+out:
-+ mutex_unlock(&zl_order_mutex);
-+ return ret;
-+}
-+
-+
-+#define MAX_NODE_LOAD (nr_online_nodes)
-+static int node_load[MAX_NUMNODES];
-+
-+/**
-+ * find_next_best_node - find the next node that should appear in a given node's fallback list
-+ * @node: node whose fallback list we're appending
-+ * @used_node_mask: nodemask_t of already used nodes
-+ *
-+ * We use a number of factors to determine which is the next node that should
-+ * appear on a given node's fallback list. The node should not have appeared
-+ * already in @node's fallback list, and it should be the next closest node
-+ * according to the distance array (which contains arbitrary distance values
-+ * from each node to each node in the system), and should also prefer nodes
-+ * with no CPUs, since presumably they'll have very little allocation pressure
-+ * on them otherwise.
-+ * It returns -1 if no node is found.
-+ */
-+static int find_next_best_node(int node, nodemask_t *used_node_mask)
-+{
-+ int n, val;
-+ int min_val = INT_MAX;
-+ int best_node = NUMA_NO_NODE;
-+ const struct cpumask *tmp = cpumask_of_node(0);
-+
-+ /* Use the local node if we haven't already */
-+ if (!node_isset(node, *used_node_mask)) {
-+ node_set(node, *used_node_mask);
-+ return node;
-+ }
-+
-+ for_each_node_state(n, N_MEMORY) {
-+
-+ /* Don't want a node to appear more than once */
-+ if (node_isset(n, *used_node_mask))
-+ continue;
-+
-+ /* Use the distance array to find the distance */
-+ val = node_distance(node, n);
-+
-+ /* Penalize nodes under us ("prefer the next node") */
-+ val += (n < node);
-+
-+ /* Give preference to headless and unused nodes */
-+ tmp = cpumask_of_node(n);
-+ if (!cpumask_empty(tmp))
-+ val += PENALTY_FOR_NODE_WITH_CPUS;
-+
-+ /* Slight preference for less loaded node */
-+ val *= (MAX_NODE_LOAD*MAX_NUMNODES);
-+ val += node_load[n];
-+
-+ if (val < min_val) {
-+ min_val = val;
-+ best_node = n;
-+ }
-+ }
-+
-+ if (best_node >= 0)
-+ node_set(best_node, *used_node_mask);
-+
-+ return best_node;
-+}
-+
-+
-+/*
-+ * Build zonelists ordered by node and zones within node.
-+ * This results in maximum locality--normal zone overflows into local
-+ * DMA zone, if any--but risks exhausting DMA zone.
-+ */
-+static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
-+{
-+ int j;
-+ struct zonelist *zonelist;
-+
-+ zonelist = &pgdat->node_zonelists[0];
-+ for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
-+ ;
-+ j = build_zonelists_node(NODE_DATA(node), zonelist, j);
-+ zonelist->_zonerefs[j].zone = NULL;
-+ zonelist->_zonerefs[j].zone_idx = 0;
-+}
-+
-+/*
-+ * Build gfp_thisnode zonelists
-+ */
-+static void build_thisnode_zonelists(pg_data_t *pgdat)
-+{
-+ int j;
-+ struct zonelist *zonelist;
-+
-+ zonelist = &pgdat->node_zonelists[1];
-+ j = build_zonelists_node(pgdat, zonelist, 0);
-+ zonelist->_zonerefs[j].zone = NULL;
-+ zonelist->_zonerefs[j].zone_idx = 0;
-+}
-+
-+/*
-+ * Build zonelists ordered by zone and nodes within zones.
-+ * This results in conserving DMA zone[s] until all Normal memory is
-+ * exhausted, but results in overflowing to remote node while memory
-+ * may still exist in local DMA zone.
-+ */
-+static int node_order[MAX_NUMNODES];
-+
-+static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
-+{
-+ int pos, j, node;
-+ int zone_type; /* needs to be signed */
-+ struct zone *z;
-+ struct zonelist *zonelist;
-+
-+ zonelist = &pgdat->node_zonelists[0];
-+ pos = 0;
-+ for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
-+ for (j = 0; j < nr_nodes; j++) {
-+ node = node_order[j];
-+ z = &NODE_DATA(node)->node_zones[zone_type];
-+ if (populated_zone(z)) {
-+ zoneref_set_zone(z,
-+ &zonelist->_zonerefs[pos++]);
-+ check_highest_zone(zone_type);
-+ }
-+ }
-+ }
-+ zonelist->_zonerefs[pos].zone = NULL;
-+ zonelist->_zonerefs[pos].zone_idx = 0;
-+}
-+
-+#if defined(CONFIG_64BIT)
-+/*
-+ * Devices that require DMA32/DMA are relatively rare and do not justify a
-+ * penalty to every machine in case the specialised case applies. Default
-+ * to Node-ordering on 64-bit NUMA machines
-+ */
-+static int default_zonelist_order(void)
-+{
-+ return ZONELIST_ORDER_NODE;
-+}
-+#else
-+/*
-+ * On 32-bit, the Normal zone needs to be preserved for allocations accessible
-+ * by the kernel. If processes running on node 0 deplete the low memory zone
-+ * then reclaim will occur more frequency increasing stalls and potentially
-+ * be easier to OOM if a large percentage of the zone is under writeback or
-+ * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set.
-+ * Hence, default to zone ordering on 32-bit.
-+ */
-+static int default_zonelist_order(void)
-+{
-+ return ZONELIST_ORDER_ZONE;
-+}
-+#endif /* CONFIG_64BIT */
-+
-+static void set_zonelist_order(void)
-+{
-+ if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
-+ current_zonelist_order = default_zonelist_order();
-+ else
-+ current_zonelist_order = user_zonelist_order;
-+}
-+
-+static void build_zonelists(pg_data_t *pgdat)
-+{
-+ int j, node, load;
-+ enum zone_type i;
-+ nodemask_t used_mask;
-+ int local_node, prev_node;
-+ struct zonelist *zonelist;
-+ int order = current_zonelist_order;
-+
-+ /* initialize zonelists */
-+ for (i = 0; i < MAX_ZONELISTS; i++) {
-+ zonelist = pgdat->node_zonelists + i;
-+ zonelist->_zonerefs[0].zone = NULL;
-+ zonelist->_zonerefs[0].zone_idx = 0;
-+ }
-+
-+ /* NUMA-aware ordering of nodes */
-+ local_node = pgdat->node_id;
-+ load = nr_online_nodes;
-+ prev_node = local_node;
-+ nodes_clear(used_mask);
-+
-+ memset(node_order, 0, sizeof(node_order));
-+ j = 0;
-+
-+ while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
-+ /*
-+ * We don't want to pressure a particular node.
-+ * So adding penalty to the first node in same
-+ * distance group to make it round-robin.
-+ */
-+ if (node_distance(local_node, node) !=
-+ node_distance(local_node, prev_node))
-+ node_load[node] = load;
-+
-+ prev_node = node;
-+ load--;
-+ if (order == ZONELIST_ORDER_NODE)
-+ build_zonelists_in_node_order(pgdat, node);
-+ else
-+ node_order[j++] = node; /* remember order */
-+ }
-+
-+ if (order == ZONELIST_ORDER_ZONE) {
-+ /* calculate node order -- i.e., DMA last! */
-+ build_zonelists_in_zone_order(pgdat, j);
-+ }
-+
-+ build_thisnode_zonelists(pgdat);
-+}
-+
-+/* Construct the zonelist performance cache - see further mmzone.h */
-+static void build_zonelist_cache(pg_data_t *pgdat)
-+{
-+ struct zonelist *zonelist;
-+ struct zonelist_cache *zlc;
-+ struct zoneref *z;
-+
-+ zonelist = &pgdat->node_zonelists[0];
-+ zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
-+ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
-+ for (z = zonelist->_zonerefs; z->zone; z++)
-+ zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
-+}
-+
-+#ifdef CONFIG_HAVE_MEMORYLESS_NODES
-+/*
-+ * Return node id of node used for "local" allocations.
-+ * I.e., first node id of first zone in arg node's generic zonelist.
-+ * Used for initializing percpu 'numa_mem', which is used primarily
-+ * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
-+ */
-+int local_memory_node(int node)
-+{
-+ struct zone *zone;
-+
-+ (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
-+ gfp_zone(GFP_KERNEL),
-+ NULL,
-+ &zone);
-+ return zone->node;
-+}
-+#endif
-+
-+#else /* CONFIG_NUMA */
-+
-+static void set_zonelist_order(void)
-+{
-+ current_zonelist_order = ZONELIST_ORDER_ZONE;
-+}
-+
-+static void build_zonelists(pg_data_t *pgdat)
-+{
-+ int node, local_node;
-+ enum zone_type j;
-+ struct zonelist *zonelist;
-+
-+ local_node = pgdat->node_id;
-+
-+ zonelist = &pgdat->node_zonelists[0];
-+ j = build_zonelists_node(pgdat, zonelist, 0);
-+
-+ /*
-+ * Now we build the zonelist so that it contains the zones
-+ * of all the other nodes.
-+ * We don't want to pressure a particular node, so when
-+ * building the zones for node N, we make sure that the
-+ * zones coming right after the local ones are those from
-+ * node N+1 (modulo N)
-+ */
-+ for (node = local_node + 1; node < MAX_NUMNODES; node++) {
-+ if (!node_online(node))
-+ continue;
-+ j = build_zonelists_node(NODE_DATA(node), zonelist, j);
-+ }
-+ for (node = 0; node < local_node; node++) {
-+ if (!node_online(node))
-+ continue;
-+ j = build_zonelists_node(NODE_DATA(node), zonelist, j);
-+ }
-+
-+ zonelist->_zonerefs[j].zone = NULL;
-+ zonelist->_zonerefs[j].zone_idx = 0;
-+}
-+
-+/* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
-+static void build_zonelist_cache(pg_data_t *pgdat)
-+{
-+ pgdat->node_zonelists[0].zlcache_ptr = NULL;
-+}
-+
-+#endif /* CONFIG_NUMA */
-+
-+/*
-+ * Boot pageset table. One per cpu which is going to be used for all
-+ * zones and all nodes. The parameters will be set in such a way
-+ * that an item put on a list will immediately be handed over to
-+ * the buddy list. This is safe since pageset manipulation is done
-+ * with interrupts disabled.
-+ *
-+ * The boot_pagesets must be kept even after bootup is complete for
-+ * unused processors and/or zones. They do play a role for bootstrapping
-+ * hotplugged processors.
-+ *
-+ * zoneinfo_show() and maybe other functions do
-+ * not check if the processor is online before following the pageset pointer.
-+ * Other parts of the kernel may not check if the zone is available.
-+ */
-+static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
-+static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
-+static void setup_zone_pageset(struct zone *zone);
-+
-+/*
-+ * Global mutex to protect against size modification of zonelists
-+ * as well as to serialize pageset setup for the new populated zone.
-+ */
-+DEFINE_MUTEX(zonelists_mutex);
-+
-+/* return values int ....just for stop_machine() */
-+static int __build_all_zonelists(void *data)
-+{
-+ int nid;
-+ int cpu;
-+ pg_data_t *self = data;
-+
-+#ifdef CONFIG_NUMA
-+ memset(node_load, 0, sizeof(node_load));
-+#endif
-+
-+ if (self && !node_online(self->node_id)) {
-+ build_zonelists(self);
-+ build_zonelist_cache(self);
-+ }
-+
-+ for_each_online_node(nid) {
-+ pg_data_t *pgdat = NODE_DATA(nid);
-+
-+ build_zonelists(pgdat);
-+ build_zonelist_cache(pgdat);
-+ }
-+
-+ /*
-+ * Initialize the boot_pagesets that are going to be used
-+ * for bootstrapping processors. The real pagesets for
-+ * each zone will be allocated later when the per cpu
-+ * allocator is available.
-+ *
-+ * boot_pagesets are used also for bootstrapping offline
-+ * cpus if the system is already booted because the pagesets
-+ * are needed to initialize allocators on a specific cpu too.
-+ * F.e. the percpu allocator needs the page allocator which
-+ * needs the percpu allocator in order to allocate its pagesets
-+ * (a chicken-egg dilemma).
-+ */
-+ for_each_possible_cpu(cpu) {
-+ setup_pageset(&per_cpu(boot_pageset, cpu), 0);
-+
-+#ifdef CONFIG_HAVE_MEMORYLESS_NODES
-+ /*
-+ * We now know the "local memory node" for each node--
-+ * i.e., the node of the first zone in the generic zonelist.
-+ * Set up numa_mem percpu variable for on-line cpus. During
-+ * boot, only the boot cpu should be on-line; we'll init the
-+ * secondary cpus' numa_mem as they come on-line. During
-+ * node/memory hotplug, we'll fixup all on-line cpus.
-+ */
-+ if (cpu_online(cpu))
-+ set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
-+#endif
-+ }
-+
-+ return 0;
-+}
-+
-+static noinline void __init
-+build_all_zonelists_init(void)
-+{
-+ __build_all_zonelists(NULL);
-+ mminit_verify_zonelist();
-+ cpuset_init_current_mems_allowed();
-+}
-+
-+/*
-+ * Called with zonelists_mutex held always
-+ * unless system_state == SYSTEM_BOOTING.
-+ *
-+ * __ref due to (1) call of __meminit annotated setup_zone_pageset
-+ * [we're only called with non-NULL zone through __meminit paths] and
-+ * (2) call of __init annotated helper build_all_zonelists_init
-+ * [protected by SYSTEM_BOOTING].
-+ */
-+void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone)
-+{
-+ set_zonelist_order();
-+
-+ if (system_state == SYSTEM_BOOTING) {
-+ build_all_zonelists_init();
-+ } else {
-+#ifdef CONFIG_MEMORY_HOTPLUG
-+ if (zone)
-+ setup_zone_pageset(zone);
-+#endif
-+ /* we have to stop all cpus to guarantee there is no user
-+ of zonelist */
-+ stop_machine(__build_all_zonelists, pgdat, NULL);
-+ /* cpuset refresh routine should be here */
-+ }
-+ vm_total_pages = nr_free_pagecache_pages();
-+ /*
-+ * Disable grouping by mobility if the number of pages in the
-+ * system is too low to allow the mechanism to work. It would be
-+ * more accurate, but expensive to check per-zone. This check is
-+ * made on memory-hotadd so a system can start with mobility
-+ * disabled and enable it later
-+ */
-+ if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
-+ page_group_by_mobility_disabled = 1;
-+ else
-+ page_group_by_mobility_disabled = 0;
-+
-+ pr_info("Built %i zonelists in %s order, mobility grouping %s. "
-+ "Total pages: %ld\n",
-+ nr_online_nodes,
-+ zonelist_order_name[current_zonelist_order],
-+ page_group_by_mobility_disabled ? "off" : "on",
-+ vm_total_pages);
-+#ifdef CONFIG_NUMA
-+ pr_info("Policy zone: %s\n", zone_names[policy_zone]);
-+#endif
-+}
-+
-+/*
-+ * Helper functions to size the waitqueue hash table.
-+ * Essentially these want to choose hash table sizes sufficiently
-+ * large so that collisions trying to wait on pages are rare.
-+ * But in fact, the number of active page waitqueues on typical
-+ * systems is ridiculously low, less than 200. So this is even
-+ * conservative, even though it seems large.
-+ *
-+ * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
-+ * waitqueues, i.e. the size of the waitq table given the number of pages.
-+ */
-+#define PAGES_PER_WAITQUEUE 256
-+
-+#ifndef CONFIG_MEMORY_HOTPLUG
-+static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
-+{
-+ unsigned long size = 1;
-+
-+ pages /= PAGES_PER_WAITQUEUE;
-+
-+ while (size < pages)
-+ size <<= 1;
-+
-+ /*
-+ * Once we have dozens or even hundreds of threads sleeping
-+ * on IO we've got bigger problems than wait queue collision.
-+ * Limit the size of the wait table to a reasonable size.
-+ */
-+ size = min(size, 4096UL);
-+
-+ return max(size, 4UL);
-+}
-+#else
-+/*
-+ * A zone's size might be changed by hot-add, so it is not possible to determine
-+ * a suitable size for its wait_table. So we use the maximum size now.
-+ *
-+ * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
-+ *
-+ * i386 (preemption config) : 4096 x 16 = 64Kbyte.
-+ * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
-+ * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
-+ *
-+ * The maximum entries are prepared when a zone's memory is (512K + 256) pages
-+ * or more by the traditional way. (See above). It equals:
-+ *
-+ * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
-+ * ia64(16K page size) : = ( 8G + 4M)byte.
-+ * powerpc (64K page size) : = (32G +16M)byte.
-+ */
-+static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
-+{
-+ return 4096UL;
-+}
-+#endif
-+
-+/*
-+ * This is an integer logarithm so that shifts can be used later
-+ * to extract the more random high bits from the multiplicative
-+ * hash function before the remainder is taken.
-+ */
-+static inline unsigned long wait_table_bits(unsigned long size)
-+{
-+ return ffz(~size);
-+}
-+
-+/*
-+ * Check if a pageblock contains reserved pages
-+ */
-+static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn)
-+{
-+ unsigned long pfn;
-+
-+ for (pfn = start_pfn; pfn < end_pfn; pfn++) {
-+ if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn)))
-+ return 1;
-+ }
-+ return 0;
-+}
-+
-+/*
-+ * Mark a number of pageblocks as MIGRATE_RESERVE. The number
-+ * of blocks reserved is based on min_wmark_pages(zone). The memory within
-+ * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
-+ * higher will lead to a bigger reserve which will get freed as contiguous
-+ * blocks as reclaim kicks in
-+ */
-+static void setup_zone_migrate_reserve(struct zone *zone)
-+{
-+ unsigned long start_pfn, pfn, end_pfn, block_end_pfn;
-+ struct page *page;
-+ unsigned long block_migratetype;
-+ int reserve;
-+ int old_reserve;
-+
-+ /*
-+ * Get the start pfn, end pfn and the number of blocks to reserve
-+ * We have to be careful to be aligned to pageblock_nr_pages to
-+ * make sure that we always check pfn_valid for the first page in
-+ * the block.
-+ */
-+ start_pfn = zone->zone_start_pfn;
-+ end_pfn = zone_end_pfn(zone);
-+ start_pfn = roundup(start_pfn, pageblock_nr_pages);
-+ reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
-+ pageblock_order;
-+
-+ /*
-+ * Reserve blocks are generally in place to help high-order atomic
-+ * allocations that are short-lived. A min_free_kbytes value that
-+ * would result in more than 2 reserve blocks for atomic allocations
-+ * is assumed to be in place to help anti-fragmentation for the
-+ * future allocation of hugepages at runtime.
-+ */
-+ reserve = min(2, reserve);
-+ old_reserve = zone->nr_migrate_reserve_block;
-+
-+ /* When memory hot-add, we almost always need to do nothing */
-+ if (reserve == old_reserve)
-+ return;
-+ zone->nr_migrate_reserve_block = reserve;
-+
-+ for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
-+ if (!pfn_valid(pfn))
-+ continue;
-+ page = pfn_to_page(pfn);
-+
-+ /* Watch out for overlapping nodes */
-+ if (page_to_nid(page) != zone_to_nid(zone))
-+ continue;
-+
-+ block_migratetype = get_pageblock_migratetype(page);
-+
-+ /* Only test what is necessary when the reserves are not met */
-+ if (reserve > 0) {
-+ /*
-+ * Blocks with reserved pages will never free, skip
-+ * them.
-+ */
-+ block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn);
-+ if (pageblock_is_reserved(pfn, block_end_pfn))
-+ continue;
-+
-+ /* If this block is reserved, account for it */
-+ if (block_migratetype == MIGRATE_RESERVE) {
-+ reserve--;
-+ continue;
-+ }
-+
-+ /* Suitable for reserving if this block is movable */
-+ if (block_migratetype == MIGRATE_MOVABLE) {
-+ set_pageblock_migratetype(page,
-+ MIGRATE_RESERVE);
-+ move_freepages_block(zone, page,
-+ MIGRATE_RESERVE);
-+ reserve--;
-+ continue;
-+ }
-+ } else if (!old_reserve) {
-+ /*
-+ * At boot time we don't need to scan the whole zone
-+ * for turning off MIGRATE_RESERVE.
-+ */
-+ break;
-+ }
-+
-+ /*
-+ * If the reserve is met and this is a previous reserved block,
-+ * take it back
-+ */
-+ if (block_migratetype == MIGRATE_RESERVE) {
-+ set_pageblock_migratetype(page, MIGRATE_MOVABLE);
-+ move_freepages_block(zone, page, MIGRATE_MOVABLE);
-+ }
-+ }
-+}
-+
-+/*
-+ * Initially all pages are reserved - free ones are freed
-+ * up by free_all_bootmem() once the early boot process is
-+ * done. Non-atomic initialization, single-pass.
-+ */
-+void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
-+ unsigned long start_pfn, enum memmap_context context)
-+{
-+ struct page *page;
-+ unsigned long end_pfn = start_pfn + size;
-+ unsigned long pfn;
-+ struct zone *z;
-+
-+ if (highest_memmap_pfn < end_pfn - 1)
-+ highest_memmap_pfn = end_pfn - 1;
-+
-+ z = &NODE_DATA(nid)->node_zones[zone];
-+ for (pfn = start_pfn; pfn < end_pfn; pfn++) {
-+ /*
-+ * There can be holes in boot-time mem_map[]s
-+ * handed to this function. They do not
-+ * exist on hotplugged memory.
-+ */
-+ if (context == MEMMAP_EARLY) {
-+ if (!early_pfn_valid(pfn))
-+ continue;
-+ if (!early_pfn_in_nid(pfn, nid))
-+ continue;
-+ }
-+ page = pfn_to_page(pfn);
-+ set_page_links(page, zone, nid, pfn);
-+ mminit_verify_page_links(page, zone, nid, pfn);
-+ init_page_count(page);
-+ page_mapcount_reset(page);
-+ page_cpupid_reset_last(page);
-+ SetPageReserved(page);
-+ /*
-+ * Mark the block movable so that blocks are reserved for
-+ * movable at startup. This will force kernel allocations
-+ * to reserve their blocks rather than leaking throughout
-+ * the address space during boot when many long-lived
-+ * kernel allocations are made. Later some blocks near
-+ * the start are marked MIGRATE_RESERVE by
-+ * setup_zone_migrate_reserve()
-+ *
-+ * bitmap is created for zone's valid pfn range. but memmap
-+ * can be created for invalid pages (for alignment)
-+ * check here not to call set_pageblock_migratetype() against
-+ * pfn out of zone.
-+ */
-+ if ((z->zone_start_pfn <= pfn)
-+ && (pfn < zone_end_pfn(z))
-+ && !(pfn & (pageblock_nr_pages - 1)))
-+ set_pageblock_migratetype(page, MIGRATE_MOVABLE);
-+
-+ INIT_LIST_HEAD(&page->lru);
-+#ifdef WANT_PAGE_VIRTUAL
-+ /* The shift won't overflow because ZONE_NORMAL is below 4G. */
-+ if (!is_highmem_idx(zone))
-+ set_page_address(page, __va(pfn << PAGE_SHIFT));
-+#endif
-+ }
-+}
-+
-+static void __meminit zone_init_free_lists(struct zone *zone)
-+{
-+ unsigned int order, t;
-+ for_each_migratetype_order(order, t) {
-+ INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
-+ zone->free_area[order].nr_free = 0;
-+ }
-+}
-+
-+#ifndef __HAVE_ARCH_MEMMAP_INIT
-+#define memmap_init(size, nid, zone, start_pfn) \
-+ memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
-+#endif
-+
-+static int zone_batchsize(struct zone *zone)
-+{
-+#ifdef CONFIG_MMU
-+ int batch;
-+
-+ /*
-+ * The per-cpu-pages pools are set to around 1000th of the
-+ * size of the zone. But no more than 1/2 of a meg.
-+ *
-+ * OK, so we don't know how big the cache is. So guess.
-+ */
-+ batch = zone->managed_pages / 1024;
-+ if (batch * PAGE_SIZE > 512 * 1024)
-+ batch = (512 * 1024) / PAGE_SIZE;
-+ batch /= 4; /* We effectively *= 4 below */
-+ if (batch < 1)
-+ batch = 1;
-+
-+ /*
-+ * Clamp the batch to a 2^n - 1 value. Having a power
-+ * of 2 value was found to be more likely to have
-+ * suboptimal cache aliasing properties in some cases.
-+ *
-+ * For example if 2 tasks are alternately allocating
-+ * batches of pages, one task can end up with a lot
-+ * of pages of one half of the possible page colors
-+ * and the other with pages of the other colors.
-+ */
-+ batch = rounddown_pow_of_two(batch + batch/2) - 1;
-+
-+ return batch;
-+
-+#else
-+ /* The deferral and batching of frees should be suppressed under NOMMU
-+ * conditions.
-+ *
-+ * The problem is that NOMMU needs to be able to allocate large chunks
-+ * of contiguous memory as there's no hardware page translation to
-+ * assemble apparent contiguous memory from discontiguous pages.
-+ *
-+ * Queueing large contiguous runs of pages for batching, however,
-+ * causes the pages to actually be freed in smaller chunks. As there
-+ * can be a significant delay between the individual batches being
-+ * recycled, this leads to the once large chunks of space being
-+ * fragmented and becoming unavailable for high-order allocations.
-+ */
-+ return 0;
-+#endif
-+}
-+
-+/*
-+ * pcp->high and pcp->batch values are related and dependent on one another:
-+ * ->batch must never be higher then ->high.
-+ * The following function updates them in a safe manner without read side
-+ * locking.
-+ *
-+ * Any new users of pcp->batch and pcp->high should ensure they can cope with
-+ * those fields changing asynchronously (acording the the above rule).
-+ *
-+ * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
-+ * outside of boot time (or some other assurance that no concurrent updaters
-+ * exist).
-+ */
-+static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
-+ unsigned long batch)
-+{
-+ /* start with a fail safe value for batch */
-+ pcp->batch = 1;
-+ smp_wmb();
-+
-+ /* Update high, then batch, in order */
-+ pcp->high = high;
-+ smp_wmb();
-+
-+ pcp->batch = batch;
-+}
-+
-+/* a companion to pageset_set_high() */
-+static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
-+{
-+ pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
-+}
-+
-+static void pageset_init(struct per_cpu_pageset *p)
-+{
-+ struct per_cpu_pages *pcp;
-+ int migratetype;
-+
-+ memset(p, 0, sizeof(*p));
-+
-+ pcp = &p->pcp;
-+ pcp->count = 0;
-+ for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
-+ INIT_LIST_HEAD(&pcp->lists[migratetype]);
-+}
-+
-+static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
-+{
-+ pageset_init(p);
-+ pageset_set_batch(p, batch);
-+}
-+
-+/*
-+ * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
-+ * to the value high for the pageset p.
-+ */
-+static void pageset_set_high(struct per_cpu_pageset *p,
-+ unsigned long high)
-+{
-+ unsigned long batch = max(1UL, high / 4);
-+ if ((high / 4) > (PAGE_SHIFT * 8))
-+ batch = PAGE_SHIFT * 8;
-+
-+ pageset_update(&p->pcp, high, batch);
-+}
-+
-+static void pageset_set_high_and_batch(struct zone *zone,
-+ struct per_cpu_pageset *pcp)
-+{
-+ if (percpu_pagelist_fraction)
-+ pageset_set_high(pcp,
-+ (zone->managed_pages /
-+ percpu_pagelist_fraction));
-+ else
-+ pageset_set_batch(pcp, zone_batchsize(zone));
-+}
-+
-+static void __meminit zone_pageset_init(struct zone *zone, int cpu)
-+{
-+ struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
-+
-+ pageset_init(pcp);
-+ pageset_set_high_and_batch(zone, pcp);
-+}
-+
-+static void __meminit setup_zone_pageset(struct zone *zone)
-+{
-+ int cpu;
-+ zone->pageset = alloc_percpu(struct per_cpu_pageset);
-+ for_each_possible_cpu(cpu)
-+ zone_pageset_init(zone, cpu);
-+}
-+
-+/*
-+ * Allocate per cpu pagesets and initialize them.
-+ * Before this call only boot pagesets were available.
-+ */
-+void __init setup_per_cpu_pageset(void)
-+{
-+ struct zone *zone;
-+
-+ for_each_populated_zone(zone)
-+ setup_zone_pageset(zone);
-+}
-+
-+static noinline __init_refok
-+int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
-+{
-+ int i;
-+ size_t alloc_size;
-+
-+ /*
-+ * The per-page waitqueue mechanism uses hashed waitqueues
-+ * per zone.
-+ */
-+ zone->wait_table_hash_nr_entries =
-+ wait_table_hash_nr_entries(zone_size_pages);
-+ zone->wait_table_bits =
-+ wait_table_bits(zone->wait_table_hash_nr_entries);
-+ alloc_size = zone->wait_table_hash_nr_entries
-+ * sizeof(wait_queue_head_t);
-+
-+ if (!slab_is_available()) {
-+ zone->wait_table = (wait_queue_head_t *)
-+ memblock_virt_alloc_node_nopanic(
-+ alloc_size, zone->zone_pgdat->node_id);
-+ } else {
-+ /*
-+ * This case means that a zone whose size was 0 gets new memory
-+ * via memory hot-add.
-+ * But it may be the case that a new node was hot-added. In
-+ * this case vmalloc() will not be able to use this new node's
-+ * memory - this wait_table must be initialized to use this new
-+ * node itself as well.
-+ * To use this new node's memory, further consideration will be
-+ * necessary.
-+ */
-+ zone->wait_table = vmalloc(alloc_size);
-+ }
-+ if (!zone->wait_table)
-+ return -ENOMEM;
-+
-+ for (i = 0; i < zone->wait_table_hash_nr_entries; ++i)
-+ init_waitqueue_head(zone->wait_table + i);
-+
-+ return 0;
-+}
-+
-+static __meminit void zone_pcp_init(struct zone *zone)
-+{
-+ /*
-+ * per cpu subsystem is not up at this point. The following code
-+ * relies on the ability of the linker to provide the
-+ * offset of a (static) per cpu variable into the per cpu area.
-+ */
-+ zone->pageset = &boot_pageset;
-+
-+ if (populated_zone(zone))
-+ printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
-+ zone->name, zone->present_pages,
-+ zone_batchsize(zone));
-+}
-+
-+int __meminit init_currently_empty_zone(struct zone *zone,
-+ unsigned long zone_start_pfn,
-+ unsigned long size,
-+ enum memmap_context context)
-+{
-+ struct pglist_data *pgdat = zone->zone_pgdat;
-+ int ret;
-+ ret = zone_wait_table_init(zone, size);
-+ if (ret)
-+ return ret;
-+ pgdat->nr_zones = zone_idx(zone) + 1;
-+
-+ zone->zone_start_pfn = zone_start_pfn;
-+
-+ mminit_dprintk(MMINIT_TRACE, "memmap_init",
-+ "Initialising map node %d zone %lu pfns %lu -> %lu\n",
-+ pgdat->node_id,
-+ (unsigned long)zone_idx(zone),
-+ zone_start_pfn, (zone_start_pfn + size));
-+
-+ zone_init_free_lists(zone);
-+
-+ return 0;
-+}
-+
-+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
-+#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
-+/*
-+ * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
-+ */
-+int __meminit __early_pfn_to_nid(unsigned long pfn)
-+{
-+ unsigned long start_pfn, end_pfn;
-+ int nid;
-+ /*
-+ * NOTE: The following SMP-unsafe globals are only used early in boot
-+ * when the kernel is running single-threaded.
-+ */
-+ static unsigned long __meminitdata last_start_pfn, last_end_pfn;
-+ static int __meminitdata last_nid;
-+
-+ if (last_start_pfn <= pfn && pfn < last_end_pfn)
-+ return last_nid;
-+
-+ nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
-+ if (nid != -1) {
-+ last_start_pfn = start_pfn;
-+ last_end_pfn = end_pfn;
-+ last_nid = nid;
-+ }
-+
-+ return nid;
-+}
-+#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
-+
-+int __meminit early_pfn_to_nid(unsigned long pfn)
-+{
-+ int nid;
-+
-+ nid = __early_pfn_to_nid(pfn);
-+ if (nid >= 0)
-+ return nid;
-+ /* just returns 0 */
-+ return 0;
-+}
-+
-+#ifdef CONFIG_NODES_SPAN_OTHER_NODES
-+bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
-+{
-+ int nid;
-+
-+ nid = __early_pfn_to_nid(pfn);
-+ if (nid >= 0 && nid != node)
-+ return false;
-+ return true;
-+}
-+#endif
-+
-+/**
-+ * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range
-+ * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
-+ * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid
-+ *
-+ * If an architecture guarantees that all ranges registered contain no holes
-+ * and may be freed, this this function may be used instead of calling
-+ * memblock_free_early_nid() manually.
-+ */
-+void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
-+{
-+ unsigned long start_pfn, end_pfn;
-+ int i, this_nid;
-+
-+ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
-+ start_pfn = min(start_pfn, max_low_pfn);
-+ end_pfn = min(end_pfn, max_low_pfn);
-+
-+ if (start_pfn < end_pfn)
-+ memblock_free_early_nid(PFN_PHYS(start_pfn),
-+ (end_pfn - start_pfn) << PAGE_SHIFT,
-+ this_nid);
-+ }
-+}
-+
-+/**
-+ * sparse_memory_present_with_active_regions - Call memory_present for each active range
-+ * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
-+ *
-+ * If an architecture guarantees that all ranges registered contain no holes and may
-+ * be freed, this function may be used instead of calling memory_present() manually.
-+ */
-+void __init sparse_memory_present_with_active_regions(int nid)
-+{
-+ unsigned long start_pfn, end_pfn;
-+ int i, this_nid;
-+
-+ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
-+ memory_present(this_nid, start_pfn, end_pfn);
-+}
-+
-+/**
-+ * get_pfn_range_for_nid - Return the start and end page frames for a node
-+ * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
-+ * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
-+ * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
-+ *
-+ * It returns the start and end page frame of a node based on information
-+ * provided by memblock_set_node(). If called for a node
-+ * with no available memory, a warning is printed and the start and end
-+ * PFNs will be 0.
-+ */
-+void __meminit get_pfn_range_for_nid(unsigned int nid,
-+ unsigned long *start_pfn, unsigned long *end_pfn)
-+{
-+ unsigned long this_start_pfn, this_end_pfn;
-+ int i;
-+
-+ *start_pfn = -1UL;
-+ *end_pfn = 0;
-+
-+ for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
-+ *start_pfn = min(*start_pfn, this_start_pfn);
-+ *end_pfn = max(*end_pfn, this_end_pfn);
-+ }
-+
-+ if (*start_pfn == -1UL)
-+ *start_pfn = 0;
-+}
-+
-+/*
-+ * This finds a zone that can be used for ZONE_MOVABLE pages. The
-+ * assumption is made that zones within a node are ordered in monotonic
-+ * increasing memory addresses so that the "highest" populated zone is used
-+ */
-+static void __init find_usable_zone_for_movable(void)
-+{
-+ int zone_index;
-+ for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
-+ if (zone_index == ZONE_MOVABLE)
-+ continue;
-+
-+ if (arch_zone_highest_possible_pfn[zone_index] >
-+ arch_zone_lowest_possible_pfn[zone_index])
-+ break;
-+ }
-+
-+ VM_BUG_ON(zone_index == -1);
-+ movable_zone = zone_index;
-+}
-+
-+/*
-+ * The zone ranges provided by the architecture do not include ZONE_MOVABLE
-+ * because it is sized independent of architecture. Unlike the other zones,
-+ * the starting point for ZONE_MOVABLE is not fixed. It may be different
-+ * in each node depending on the size of each node and how evenly kernelcore
-+ * is distributed. This helper function adjusts the zone ranges
-+ * provided by the architecture for a given node by using the end of the
-+ * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
-+ * zones within a node are in order of monotonic increases memory addresses
-+ */
-+static void __meminit adjust_zone_range_for_zone_movable(int nid,
-+ unsigned long zone_type,
-+ unsigned long node_start_pfn,
-+ unsigned long node_end_pfn,
-+ unsigned long *zone_start_pfn,
-+ unsigned long *zone_end_pfn)
-+{
-+ /* Only adjust if ZONE_MOVABLE is on this node */
-+ if (zone_movable_pfn[nid]) {
-+ /* Size ZONE_MOVABLE */
-+ if (zone_type == ZONE_MOVABLE) {
-+ *zone_start_pfn = zone_movable_pfn[nid];
-+ *zone_end_pfn = min(node_end_pfn,
-+ arch_zone_highest_possible_pfn[movable_zone]);
-+
-+ /* Adjust for ZONE_MOVABLE starting within this range */
-+ } else if (*zone_start_pfn < zone_movable_pfn[nid] &&
-+ *zone_end_pfn > zone_movable_pfn[nid]) {
-+ *zone_end_pfn = zone_movable_pfn[nid];
-+
-+ /* Check if this whole range is within ZONE_MOVABLE */
-+ } else if (*zone_start_pfn >= zone_movable_pfn[nid])
-+ *zone_start_pfn = *zone_end_pfn;
-+ }
-+}
-+
-+/*
-+ * Return the number of pages a zone spans in a node, including holes
-+ * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
-+ */
-+static unsigned long __meminit zone_spanned_pages_in_node(int nid,
-+ unsigned long zone_type,
-+ unsigned long node_start_pfn,
-+ unsigned long node_end_pfn,
-+ unsigned long *ignored)
-+{
-+ unsigned long zone_start_pfn, zone_end_pfn;
-+
-+ /* Get the start and end of the zone */
-+ zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
-+ zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
-+ adjust_zone_range_for_zone_movable(nid, zone_type,
-+ node_start_pfn, node_end_pfn,
-+ &zone_start_pfn, &zone_end_pfn);
-+
-+ /* Check that this node has pages within the zone's required range */
-+ if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
-+ return 0;
-+
-+ /* Move the zone boundaries inside the node if necessary */
-+ zone_end_pfn = min(zone_end_pfn, node_end_pfn);
-+ zone_start_pfn = max(zone_start_pfn, node_start_pfn);
-+
-+ /* Return the spanned pages */
-+ return zone_end_pfn - zone_start_pfn;
-+}
-+
-+/*
-+ * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
-+ * then all holes in the requested range will be accounted for.
-+ */
-+unsigned long __meminit __absent_pages_in_range(int nid,
-+ unsigned long range_start_pfn,
-+ unsigned long range_end_pfn)
-+{
-+ unsigned long nr_absent = range_end_pfn - range_start_pfn;
-+ unsigned long start_pfn, end_pfn;
-+ int i;
-+
-+ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
-+ start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
-+ end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
-+ nr_absent -= end_pfn - start_pfn;
-+ }
-+ return nr_absent;
-+}
-+
-+/**
-+ * absent_pages_in_range - Return number of page frames in holes within a range
-+ * @start_pfn: The start PFN to start searching for holes
-+ * @end_pfn: The end PFN to stop searching for holes
-+ *
-+ * It returns the number of pages frames in memory holes within a range.
-+ */
-+unsigned long __init absent_pages_in_range(unsigned long start_pfn,
-+ unsigned long end_pfn)
-+{
-+ return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
-+}
-+
-+/* Return the number of page frames in holes in a zone on a node */
-+static unsigned long __meminit zone_absent_pages_in_node(int nid,
-+ unsigned long zone_type,
-+ unsigned long node_start_pfn,
-+ unsigned long node_end_pfn,
-+ unsigned long *ignored)
-+{
-+ unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
-+ unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
-+ unsigned long zone_start_pfn, zone_end_pfn;
-+
-+ zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
-+ zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
-+
-+ adjust_zone_range_for_zone_movable(nid, zone_type,
-+ node_start_pfn, node_end_pfn,
-+ &zone_start_pfn, &zone_end_pfn);
-+ return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
-+}
-+
-+#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
-+static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
-+ unsigned long zone_type,
-+ unsigned long node_start_pfn,
-+ unsigned long node_end_pfn,
-+ unsigned long *zones_size)
-+{
-+ return zones_size[zone_type];
-+}
-+
-+static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
-+ unsigned long zone_type,
-+ unsigned long node_start_pfn,
-+ unsigned long node_end_pfn,
-+ unsigned long *zholes_size)
-+{
-+ if (!zholes_size)
-+ return 0;
-+
-+ return zholes_size[zone_type];
-+}
-+
-+#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
-+
-+static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
-+ unsigned long node_start_pfn,
-+ unsigned long node_end_pfn,
-+ unsigned long *zones_size,
-+ unsigned long *zholes_size)
-+{
-+ unsigned long realtotalpages, totalpages = 0;
-+ enum zone_type i;
-+
-+ for (i = 0; i < MAX_NR_ZONES; i++)
-+ totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
-+ node_start_pfn,
-+ node_end_pfn,
-+ zones_size);
-+ pgdat->node_spanned_pages = totalpages;
-+
-+ realtotalpages = totalpages;
-+ for (i = 0; i < MAX_NR_ZONES; i++)
-+ realtotalpages -=
-+ zone_absent_pages_in_node(pgdat->node_id, i,
-+ node_start_pfn, node_end_pfn,
-+ zholes_size);
-+ pgdat->node_present_pages = realtotalpages;
-+ printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
-+ realtotalpages);
-+}
-+
-+#ifndef CONFIG_SPARSEMEM
-+/*
-+ * Calculate the size of the zone->blockflags rounded to an unsigned long
-+ * Start by making sure zonesize is a multiple of pageblock_order by rounding
-+ * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
-+ * round what is now in bits to nearest long in bits, then return it in
-+ * bytes.
-+ */
-+static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
-+{
-+ unsigned long usemapsize;
-+
-+ zonesize += zone_start_pfn & (pageblock_nr_pages-1);
-+ usemapsize = roundup(zonesize, pageblock_nr_pages);
-+ usemapsize = usemapsize >> pageblock_order;
-+ usemapsize *= NR_PAGEBLOCK_BITS;
-+ usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
-+
-+ return usemapsize / 8;
-+}
-+
-+static void __init setup_usemap(struct pglist_data *pgdat,
-+ struct zone *zone,
-+ unsigned long zone_start_pfn,
-+ unsigned long zonesize)
-+{
-+ unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
-+ zone->pageblock_flags = NULL;
-+ if (usemapsize)
-+ zone->pageblock_flags =
-+ memblock_virt_alloc_node_nopanic(usemapsize,
-+ pgdat->node_id);
-+}
-+#else
-+static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
-+ unsigned long zone_start_pfn, unsigned long zonesize) {}
-+#endif /* CONFIG_SPARSEMEM */
-+
-+#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
-+
-+/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
-+void __paginginit set_pageblock_order(void)
-+{
-+ unsigned int order;
-+
-+ /* Check that pageblock_nr_pages has not already been setup */
-+ if (pageblock_order)
-+ return;
-+
-+ if (HPAGE_SHIFT > PAGE_SHIFT)
-+ order = HUGETLB_PAGE_ORDER;
-+ else
-+ order = MAX_ORDER - 1;
-+
-+ /*
-+ * Assume the largest contiguous order of interest is a huge page.
-+ * This value may be variable depending on boot parameters on IA64 and
-+ * powerpc.
-+ */
-+ pageblock_order = order;
-+}
-+#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
-+
-+/*
-+ * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
-+ * is unused as pageblock_order is set at compile-time. See
-+ * include/linux/pageblock-flags.h for the values of pageblock_order based on
-+ * the kernel config
-+ */
-+void __paginginit set_pageblock_order(void)
-+{
-+}
-+
-+#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
-+
-+static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages,
-+ unsigned long present_pages)
-+{
-+ unsigned long pages = spanned_pages;
-+
-+ /*
-+ * Provide a more accurate estimation if there are holes within
-+ * the zone and SPARSEMEM is in use. If there are holes within the
-+ * zone, each populated memory region may cost us one or two extra
-+ * memmap pages due to alignment because memmap pages for each
-+ * populated regions may not naturally algined on page boundary.
-+ * So the (present_pages >> 4) heuristic is a tradeoff for that.
-+ */
-+ if (spanned_pages > present_pages + (present_pages >> 4) &&
-+ IS_ENABLED(CONFIG_SPARSEMEM))
-+ pages = present_pages;
-+
-+ return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
-+}
-+
-+/*
-+ * Set up the zone data structures:
-+ * - mark all pages reserved
-+ * - mark all memory queues empty
-+ * - clear the memory bitmaps
-+ *
-+ * NOTE: pgdat should get zeroed by caller.
-+ */
-+static void __paginginit free_area_init_core(struct pglist_data *pgdat,
-+ unsigned long node_start_pfn, unsigned long node_end_pfn,
-+ unsigned long *zones_size, unsigned long *zholes_size)
-+{
-+ enum zone_type j;
-+ int nid = pgdat->node_id;
-+ unsigned long zone_start_pfn = pgdat->node_start_pfn;
-+ int ret;
-+
-+ pgdat_resize_init(pgdat);
-+#ifdef CONFIG_NUMA_BALANCING
-+ spin_lock_init(&pgdat->numabalancing_migrate_lock);
-+ pgdat->numabalancing_migrate_nr_pages = 0;
-+ pgdat->numabalancing_migrate_next_window = jiffies;
-+#endif
-+ init_waitqueue_head(&pgdat->kswapd_wait);
-+ init_waitqueue_head(&pgdat->pfmemalloc_wait);
-+ pgdat_page_ext_init(pgdat);
-+
-+ for (j = 0; j < MAX_NR_ZONES; j++) {
-+ struct zone *zone = pgdat->node_zones + j;
-+ unsigned long size, realsize, freesize, memmap_pages;
-+
-+ size = zone_spanned_pages_in_node(nid, j, node_start_pfn,
-+ node_end_pfn, zones_size);
-+ realsize = freesize = size - zone_absent_pages_in_node(nid, j,
-+ node_start_pfn,
-+ node_end_pfn,
-+ zholes_size);
-+
-+ /*
-+ * Adjust freesize so that it accounts for how much memory
-+ * is used by this zone for memmap. This affects the watermark
-+ * and per-cpu initialisations
-+ */
-+ memmap_pages = calc_memmap_size(size, realsize);
-+ if (!is_highmem_idx(j)) {
-+ if (freesize >= memmap_pages) {
-+ freesize -= memmap_pages;
-+ if (memmap_pages)
-+ printk(KERN_DEBUG
-+ " %s zone: %lu pages used for memmap\n",
-+ zone_names[j], memmap_pages);
-+ } else
-+ printk(KERN_WARNING
-+ " %s zone: %lu pages exceeds freesize %lu\n",
-+ zone_names[j], memmap_pages, freesize);
-+ }
-+
-+ /* Account for reserved pages */
-+ if (j == 0 && freesize > dma_reserve) {
-+ freesize -= dma_reserve;
-+ printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
-+ zone_names[0], dma_reserve);
-+ }
-+
-+ if (!is_highmem_idx(j))
-+ nr_kernel_pages += freesize;
-+ /* Charge for highmem memmap if there are enough kernel pages */
-+ else if (nr_kernel_pages > memmap_pages * 2)
-+ nr_kernel_pages -= memmap_pages;
-+ nr_all_pages += freesize;
-+
-+ zone->spanned_pages = size;
-+ zone->present_pages = realsize;
-+ /*
-+ * Set an approximate value for lowmem here, it will be adjusted
-+ * when the bootmem allocator frees pages into the buddy system.
-+ * And all highmem pages will be managed by the buddy system.
-+ */
-+ zone->managed_pages = is_highmem_idx(j) ? realsize : freesize;
-+#ifdef CONFIG_NUMA
-+ zone->node = nid;
-+ zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio)
-+ / 100;
-+ zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100;
-+#endif
-+ zone->name = zone_names[j];
-+ spin_lock_init(&zone->lock);
-+ spin_lock_init(&zone->lru_lock);
-+ zone_seqlock_init(zone);
-+ zone->zone_pgdat = pgdat;
-+ zone_pcp_init(zone);
-+
-+ /* For bootup, initialized properly in watermark setup */
-+ mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages);
-+
-+ lruvec_init(&zone->lruvec);
-+ if (!size)
-+ continue;
-+
-+ set_pageblock_order();
-+ setup_usemap(pgdat, zone, zone_start_pfn, size);
-+ ret = init_currently_empty_zone(zone, zone_start_pfn,
-+ size, MEMMAP_EARLY);
-+ BUG_ON(ret);
-+ memmap_init(size, nid, j, zone_start_pfn);
-+ zone_start_pfn += size;
-+ }
-+}
-+
-+static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
-+{
-+ /* Skip empty nodes */
-+ if (!pgdat->node_spanned_pages)
-+ return;
-+
-+#ifdef CONFIG_FLAT_NODE_MEM_MAP
-+ /* ia64 gets its own node_mem_map, before this, without bootmem */
-+ if (!pgdat->node_mem_map) {
-+ unsigned long size, start, end;
-+ struct page *map;
-+
-+ /*
-+ * The zone's endpoints aren't required to be MAX_ORDER
-+ * aligned but the node_mem_map endpoints must be in order
-+ * for the buddy allocator to function correctly.
-+ */
-+ start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
-+ end = pgdat_end_pfn(pgdat);
-+ end = ALIGN(end, MAX_ORDER_NR_PAGES);
-+ size = (end - start) * sizeof(struct page);
-+ map = alloc_remap(pgdat->node_id, size);
-+ if (!map)
-+ map = memblock_virt_alloc_node_nopanic(size,
-+ pgdat->node_id);
-+ pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
-+ }
-+#ifndef CONFIG_NEED_MULTIPLE_NODES
-+ /*
-+ * With no DISCONTIG, the global mem_map is just set as node 0's
-+ */
-+ if (pgdat == NODE_DATA(0)) {
-+ mem_map = NODE_DATA(0)->node_mem_map;
-+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
-+ if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
-+ mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
-+#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
-+ }
-+#endif
-+#endif /* CONFIG_FLAT_NODE_MEM_MAP */
-+}
-+
-+void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
-+ unsigned long node_start_pfn, unsigned long *zholes_size)
-+{
-+ pg_data_t *pgdat = NODE_DATA(nid);
-+ unsigned long start_pfn = 0;
-+ unsigned long end_pfn = 0;
-+
-+ /* pg_data_t should be reset to zero when it's allocated */
-+ WARN_ON(pgdat->nr_zones || pgdat->classzone_idx);
-+
-+ pgdat->node_id = nid;
-+ pgdat->node_start_pfn = node_start_pfn;
-+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
-+ get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
-+ pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
-+ (u64)start_pfn << PAGE_SHIFT, ((u64)end_pfn << PAGE_SHIFT) - 1);
-+#endif
-+ calculate_node_totalpages(pgdat, start_pfn, end_pfn,
-+ zones_size, zholes_size);
-+
-+ alloc_node_mem_map(pgdat);
-+#ifdef CONFIG_FLAT_NODE_MEM_MAP
-+ printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
-+ nid, (unsigned long)pgdat,
-+ (unsigned long)pgdat->node_mem_map);
-+#endif
-+
-+ free_area_init_core(pgdat, start_pfn, end_pfn,
-+ zones_size, zholes_size);
-+}
-+
-+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
-+
-+#if MAX_NUMNODES > 1
-+/*
-+ * Figure out the number of possible node ids.
-+ */
-+void __init setup_nr_node_ids(void)
-+{
-+ unsigned int node;
-+ unsigned int highest = 0;
-+
-+ for_each_node_mask(node, node_possible_map)
-+ highest = node;
-+ nr_node_ids = highest + 1;
-+}
-+#endif
-+
-+/**
-+ * node_map_pfn_alignment - determine the maximum internode alignment
-+ *
-+ * This function should be called after node map is populated and sorted.
-+ * It calculates the maximum power of two alignment which can distinguish
-+ * all the nodes.
-+ *
-+ * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
-+ * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
-+ * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
-+ * shifted, 1GiB is enough and this function will indicate so.
-+ *
-+ * This is used to test whether pfn -> nid mapping of the chosen memory
-+ * model has fine enough granularity to avoid incorrect mapping for the
-+ * populated node map.
-+ *
-+ * Returns the determined alignment in pfn's. 0 if there is no alignment
-+ * requirement (single node).
-+ */
-+unsigned long __init node_map_pfn_alignment(void)
-+{
-+ unsigned long accl_mask = 0, last_end = 0;
-+ unsigned long start, end, mask;
-+ int last_nid = -1;
-+ int i, nid;
-+
-+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
-+ if (!start || last_nid < 0 || last_nid == nid) {
-+ last_nid = nid;
-+ last_end = end;
-+ continue;
-+ }
-+
-+ /*
-+ * Start with a mask granular enough to pin-point to the
-+ * start pfn and tick off bits one-by-one until it becomes
-+ * too coarse to separate the current node from the last.
-+ */
-+ mask = ~((1 << __ffs(start)) - 1);
-+ while (mask && last_end <= (start & (mask << 1)))
-+ mask <<= 1;
-+
-+ /* accumulate all internode masks */
-+ accl_mask |= mask;
-+ }
-+
-+ /* convert mask to number of pages */
-+ return ~accl_mask + 1;
-+}
-+
-+/* Find the lowest pfn for a node */
-+static unsigned long __init find_min_pfn_for_node(int nid)
-+{
-+ unsigned long min_pfn = ULONG_MAX;
-+ unsigned long start_pfn;
-+ int i;
-+
-+ for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
-+ min_pfn = min(min_pfn, start_pfn);
-+
-+ if (min_pfn == ULONG_MAX) {
-+ printk(KERN_WARNING
-+ "Could not find start_pfn for node %d\n", nid);
-+ return 0;
-+ }
-+
-+ return min_pfn;
-+}
-+
-+/**
-+ * find_min_pfn_with_active_regions - Find the minimum PFN registered
-+ *
-+ * It returns the minimum PFN based on information provided via
-+ * memblock_set_node().
-+ */
-+unsigned long __init find_min_pfn_with_active_regions(void)
-+{
-+ return find_min_pfn_for_node(MAX_NUMNODES);
-+}
-+
-+/*
-+ * early_calculate_totalpages()
-+ * Sum pages in active regions for movable zone.
-+ * Populate N_MEMORY for calculating usable_nodes.
-+ */
-+static unsigned long __init early_calculate_totalpages(void)
-+{
-+ unsigned long totalpages = 0;
-+ unsigned long start_pfn, end_pfn;
-+ int i, nid;
-+
-+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
-+ unsigned long pages = end_pfn - start_pfn;
-+
-+ totalpages += pages;
-+ if (pages)
-+ node_set_state(nid, N_MEMORY);
-+ }
-+ return totalpages;
-+}
-+
-+/*
-+ * Find the PFN the Movable zone begins in each node. Kernel memory
-+ * is spread evenly between nodes as long as the nodes have enough
-+ * memory. When they don't, some nodes will have more kernelcore than
-+ * others
-+ */
-+static void __init find_zone_movable_pfns_for_nodes(void)
-+{
-+ int i, nid;
-+ unsigned long usable_startpfn;
-+ unsigned long kernelcore_node, kernelcore_remaining;
-+ /* save the state before borrow the nodemask */
-+ nodemask_t saved_node_state = node_states[N_MEMORY];
-+ unsigned long totalpages = early_calculate_totalpages();
-+ int usable_nodes = nodes_weight(node_states[N_MEMORY]);
-+ struct memblock_region *r;
-+
-+ /* Need to find movable_zone earlier when movable_node is specified. */
-+ find_usable_zone_for_movable();
-+
-+ /*
-+ * If movable_node is specified, ignore kernelcore and movablecore
-+ * options.
-+ */
-+ if (movable_node_is_enabled()) {
-+ for_each_memblock(memory, r) {
-+ if (!memblock_is_hotpluggable(r))
-+ continue;
-+
-+ nid = r->nid;
-+
-+ usable_startpfn = PFN_DOWN(r->base);
-+ zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
-+ min(usable_startpfn, zone_movable_pfn[nid]) :
-+ usable_startpfn;
-+ }
-+
-+ goto out2;
-+ }
-+
-+ /*
-+ * If movablecore=nn[KMG] was specified, calculate what size of
-+ * kernelcore that corresponds so that memory usable for
-+ * any allocation type is evenly spread. If both kernelcore
-+ * and movablecore are specified, then the value of kernelcore
-+ * will be used for required_kernelcore if it's greater than
-+ * what movablecore would have allowed.
-+ */
-+ if (required_movablecore) {
-+ unsigned long corepages;
-+
-+ /*
-+ * Round-up so that ZONE_MOVABLE is at least as large as what
-+ * was requested by the user
-+ */
-+ required_movablecore =
-+ roundup(required_movablecore, MAX_ORDER_NR_PAGES);
-+ corepages = totalpages - required_movablecore;
-+
-+ required_kernelcore = max(required_kernelcore, corepages);
-+ }
-+
-+ /* If kernelcore was not specified, there is no ZONE_MOVABLE */
-+ if (!required_kernelcore)
-+ goto out;
-+
-+ /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
-+ usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
-+
-+restart:
-+ /* Spread kernelcore memory as evenly as possible throughout nodes */
-+ kernelcore_node = required_kernelcore / usable_nodes;
-+ for_each_node_state(nid, N_MEMORY) {
-+ unsigned long start_pfn, end_pfn;
-+
-+ /*
-+ * Recalculate kernelcore_node if the division per node
-+ * now exceeds what is necessary to satisfy the requested
-+ * amount of memory for the kernel
-+ */
-+ if (required_kernelcore < kernelcore_node)
-+ kernelcore_node = required_kernelcore / usable_nodes;
-+
-+ /*
-+ * As the map is walked, we track how much memory is usable
-+ * by the kernel using kernelcore_remaining. When it is
-+ * 0, the rest of the node is usable by ZONE_MOVABLE
-+ */
-+ kernelcore_remaining = kernelcore_node;
-+
-+ /* Go through each range of PFNs within this node */
-+ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
-+ unsigned long size_pages;
-+
-+ start_pfn = max(start_pfn, zone_movable_pfn[nid]);
-+ if (start_pfn >= end_pfn)
-+ continue;
-+
-+ /* Account for what is only usable for kernelcore */
-+ if (start_pfn < usable_startpfn) {
-+ unsigned long kernel_pages;
-+ kernel_pages = min(end_pfn, usable_startpfn)
-+ - start_pfn;
-+
-+ kernelcore_remaining -= min(kernel_pages,
-+ kernelcore_remaining);
-+ required_kernelcore -= min(kernel_pages,
-+ required_kernelcore);
-+
-+ /* Continue if range is now fully accounted */
-+ if (end_pfn <= usable_startpfn) {
-+
-+ /*
-+ * Push zone_movable_pfn to the end so
-+ * that if we have to rebalance
-+ * kernelcore across nodes, we will
-+ * not double account here
-+ */
-+ zone_movable_pfn[nid] = end_pfn;
-+ continue;
-+ }
-+ start_pfn = usable_startpfn;
-+ }
-+
-+ /*
-+ * The usable PFN range for ZONE_MOVABLE is from
-+ * start_pfn->end_pfn. Calculate size_pages as the
-+ * number of pages used as kernelcore
-+ */
-+ size_pages = end_pfn - start_pfn;
-+ if (size_pages > kernelcore_remaining)
-+ size_pages = kernelcore_remaining;
-+ zone_movable_pfn[nid] = start_pfn + size_pages;
-+
-+ /*
-+ * Some kernelcore has been met, update counts and
-+ * break if the kernelcore for this node has been
-+ * satisfied
-+ */
-+ required_kernelcore -= min(required_kernelcore,
-+ size_pages);
-+ kernelcore_remaining -= size_pages;
-+ if (!kernelcore_remaining)
-+ break;
-+ }
-+ }
-+
-+ /*
-+ * If there is still required_kernelcore, we do another pass with one
-+ * less node in the count. This will push zone_movable_pfn[nid] further
-+ * along on the nodes that still have memory until kernelcore is
-+ * satisfied
-+ */
-+ usable_nodes--;
-+ if (usable_nodes && required_kernelcore > usable_nodes)
-+ goto restart;
-+
-+out2:
-+ /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
-+ for (nid = 0; nid < MAX_NUMNODES; nid++)
-+ zone_movable_pfn[nid] =
-+ roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
-+
-+out:
-+ /* restore the node_state */
-+ node_states[N_MEMORY] = saved_node_state;
-+}
-+
-+/* Any regular or high memory on that node ? */
-+static void check_for_memory(pg_data_t *pgdat, int nid)
-+{
-+ enum zone_type zone_type;
-+
-+ if (N_MEMORY == N_NORMAL_MEMORY)
-+ return;
-+
-+ for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
-+ struct zone *zone = &pgdat->node_zones[zone_type];
-+ if (populated_zone(zone)) {
-+ node_set_state(nid, N_HIGH_MEMORY);
-+ if (N_NORMAL_MEMORY != N_HIGH_MEMORY &&
-+ zone_type <= ZONE_NORMAL)
-+ node_set_state(nid, N_NORMAL_MEMORY);
-+ break;
-+ }
-+ }
-+}
-+
-+/**
-+ * free_area_init_nodes - Initialise all pg_data_t and zone data
-+ * @max_zone_pfn: an array of max PFNs for each zone
-+ *
-+ * This will call free_area_init_node() for each active node in the system.
-+ * Using the page ranges provided by memblock_set_node(), the size of each
-+ * zone in each node and their holes is calculated. If the maximum PFN
-+ * between two adjacent zones match, it is assumed that the zone is empty.
-+ * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
-+ * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
-+ * starts where the previous one ended. For example, ZONE_DMA32 starts
-+ * at arch_max_dma_pfn.
-+ */
-+void __init free_area_init_nodes(unsigned long *max_zone_pfn)
-+{
-+ unsigned long start_pfn, end_pfn;
-+ int i, nid;
-+
-+ /* Record where the zone boundaries are */
-+ memset(arch_zone_lowest_possible_pfn, 0,
-+ sizeof(arch_zone_lowest_possible_pfn));
-+ memset(arch_zone_highest_possible_pfn, 0,
-+ sizeof(arch_zone_highest_possible_pfn));
-+ arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
-+ arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
-+ for (i = 1; i < MAX_NR_ZONES; i++) {
-+ if (i == ZONE_MOVABLE)
-+ continue;
-+ arch_zone_lowest_possible_pfn[i] =
-+ arch_zone_highest_possible_pfn[i-1];
-+ arch_zone_highest_possible_pfn[i] =
-+ max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
-+ }
-+ arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
-+ arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
-+
-+ /* Find the PFNs that ZONE_MOVABLE begins at in each node */
-+ memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
-+ find_zone_movable_pfns_for_nodes();
-+
-+ /* Print out the zone ranges */
-+ pr_info("Zone ranges:\n");
-+ for (i = 0; i < MAX_NR_ZONES; i++) {
-+ if (i == ZONE_MOVABLE)
-+ continue;
-+ pr_info(" %-8s ", zone_names[i]);
-+ if (arch_zone_lowest_possible_pfn[i] ==
-+ arch_zone_highest_possible_pfn[i])
-+ pr_cont("empty\n");
-+ else
-+ pr_cont("[mem %#018Lx-%#018Lx]\n",
-+ (u64)arch_zone_lowest_possible_pfn[i]
-+ << PAGE_SHIFT,
-+ ((u64)arch_zone_highest_possible_pfn[i]
-+ << PAGE_SHIFT) - 1);
-+ }
-+
-+ /* Print out the PFNs ZONE_MOVABLE begins at in each node */
-+ pr_info("Movable zone start for each node\n");
-+ for (i = 0; i < MAX_NUMNODES; i++) {
-+ if (zone_movable_pfn[i])
-+ pr_info(" Node %d: %#018Lx\n", i,
-+ (u64)zone_movable_pfn[i] << PAGE_SHIFT);
-+ }
-+
-+ /* Print out the early node map */
-+ pr_info("Early memory node ranges\n");
-+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
-+ pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
-+ (u64)start_pfn << PAGE_SHIFT,
-+ ((u64)end_pfn << PAGE_SHIFT) - 1);
-+
-+ /* Initialise every node */
-+ mminit_verify_pageflags_layout();
-+ setup_nr_node_ids();
-+ for_each_online_node(nid) {
-+ pg_data_t *pgdat = NODE_DATA(nid);
-+ free_area_init_node(nid, NULL,
-+ find_min_pfn_for_node(nid), NULL);
-+
-+ /* Any memory on that node */
-+ if (pgdat->node_present_pages)
-+ node_set_state(nid, N_MEMORY);
-+ check_for_memory(pgdat, nid);
-+ }
-+}
-+
-+static int __init cmdline_parse_core(char *p, unsigned long *core)
-+{
-+ unsigned long long coremem;
-+ if (!p)
-+ return -EINVAL;
-+
-+ coremem = memparse(p, &p);
-+ *core = coremem >> PAGE_SHIFT;
-+
-+ /* Paranoid check that UL is enough for the coremem value */
-+ WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
-+
-+ return 0;
-+}
-+
-+/*
-+ * kernelcore=size sets the amount of memory for use for allocations that
-+ * cannot be reclaimed or migrated.
-+ */
-+static int __init cmdline_parse_kernelcore(char *p)
-+{
-+ return cmdline_parse_core(p, &required_kernelcore);
-+}
-+
-+/*
-+ * movablecore=size sets the amount of memory for use for allocations that
-+ * can be reclaimed or migrated.
-+ */
-+static int __init cmdline_parse_movablecore(char *p)
-+{
-+ return cmdline_parse_core(p, &required_movablecore);
-+}
-+
-+early_param("kernelcore", cmdline_parse_kernelcore);
-+early_param("movablecore", cmdline_parse_movablecore);
-+
-+#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
-+
-+void adjust_managed_page_count(struct page *page, long count)
-+{
-+ spin_lock(&managed_page_count_lock);
-+ page_zone(page)->managed_pages += count;
-+ totalram_pages += count;
-+#ifdef CONFIG_HIGHMEM
-+ if (PageHighMem(page))
-+ totalhigh_pages += count;
-+#endif
-+ spin_unlock(&managed_page_count_lock);
-+}
-+EXPORT_SYMBOL(adjust_managed_page_count);
-+
-+unsigned long free_reserved_area(void *start, void *end, int poison, char *s)
-+{
-+ void *pos;
-+ unsigned long pages = 0;
-+
-+ start = (void *)PAGE_ALIGN((unsigned long)start);
-+ end = (void *)((unsigned long)end & PAGE_MASK);
-+ for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
-+ if ((unsigned int)poison <= 0xFF)
-+ memset(pos, poison, PAGE_SIZE);
-+ free_reserved_page(virt_to_page(pos));
-+ }
-+
-+ if (pages && s)
-+ pr_info("Freeing %s memory: %ldK (%p - %p)\n",
-+ s, pages << (PAGE_SHIFT - 10), start, end);
-+
-+ return pages;
-+}
-+EXPORT_SYMBOL(free_reserved_area);
-+
-+#ifdef CONFIG_HIGHMEM
-+void free_highmem_page(struct page *page)
-+{
-+ __free_reserved_page(page);
-+ totalram_pages++;
-+ page_zone(page)->managed_pages++;
-+ totalhigh_pages++;
-+}
-+#endif
-+
-+
-+void __init mem_init_print_info(const char *str)
-+{
-+ unsigned long physpages, codesize, datasize, rosize, bss_size;
-+ unsigned long init_code_size, init_data_size;
-+
-+ physpages = get_num_physpages();
-+ codesize = _etext - _stext;
-+ datasize = _edata - _sdata;
-+ rosize = __end_rodata - __start_rodata;
-+ bss_size = __bss_stop - __bss_start;
-+ init_data_size = __init_end - __init_begin;
-+ init_code_size = _einittext - _sinittext;
-+
-+ /*
-+ * Detect special cases and adjust section sizes accordingly:
-+ * 1) .init.* may be embedded into .data sections
-+ * 2) .init.text.* may be out of [__init_begin, __init_end],
-+ * please refer to arch/tile/kernel/vmlinux.lds.S.
-+ * 3) .rodata.* may be embedded into .text or .data sections.
-+ */
-+#define adj_init_size(start, end, size, pos, adj) \
-+ do { \
-+ if (start <= pos && pos < end && size > adj) \
-+ size -= adj; \
-+ } while (0)
-+
-+ adj_init_size(__init_begin, __init_end, init_data_size,
-+ _sinittext, init_code_size);
-+ adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
-+ adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
-+ adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
-+ adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
-+
-+#undef adj_init_size
-+
-+ pr_info("Memory: %luK/%luK available "
-+ "(%luK kernel code, %luK rwdata, %luK rodata, "
-+ "%luK init, %luK bss, %luK reserved, %luK cma-reserved"
-+#ifdef CONFIG_HIGHMEM
-+ ", %luK highmem"
-+#endif
-+ "%s%s)\n",
-+ nr_free_pages() << (PAGE_SHIFT-10), physpages << (PAGE_SHIFT-10),
-+ codesize >> 10, datasize >> 10, rosize >> 10,
-+ (init_data_size + init_code_size) >> 10, bss_size >> 10,
-+ (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT-10),
-+ totalcma_pages << (PAGE_SHIFT-10),
-+#ifdef CONFIG_HIGHMEM
-+ totalhigh_pages << (PAGE_SHIFT-10),
-+#endif
-+ str ? ", " : "", str ? str : "");
-+}
-+
-+/**
-+ * set_dma_reserve - set the specified number of pages reserved in the first zone
-+ * @new_dma_reserve: The number of pages to mark reserved
-+ *
-+ * The per-cpu batchsize and zone watermarks are determined by present_pages.
-+ * In the DMA zone, a significant percentage may be consumed by kernel image
-+ * and other unfreeable allocations which can skew the watermarks badly. This
-+ * function may optionally be used to account for unfreeable pages in the
-+ * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
-+ * smaller per-cpu batchsize.
-+ */
-+void __init set_dma_reserve(unsigned long new_dma_reserve)
-+{
-+ dma_reserve = new_dma_reserve;
-+}
-+
-+void __init free_area_init(unsigned long *zones_size)
-+{
-+ free_area_init_node(0, zones_size,
-+ __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
-+}
-+
-+static int page_alloc_cpu_notify(struct notifier_block *self,
-+ unsigned long action, void *hcpu)
-+{
-+ int cpu = (unsigned long)hcpu;
-+
-+ if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
-+ lru_add_drain_cpu(cpu);
-+ drain_pages(cpu);
-+
-+ /*
-+ * Spill the event counters of the dead processor
-+ * into the current processors event counters.
-+ * This artificially elevates the count of the current
-+ * processor.
-+ */
-+ vm_events_fold_cpu(cpu);
-+
-+ /*
-+ * Zero the differential counters of the dead processor
-+ * so that the vm statistics are consistent.
-+ *
-+ * This is only okay since the processor is dead and cannot
-+ * race with what we are doing.
-+ */
-+ cpu_vm_stats_fold(cpu);
-+ }
-+ return NOTIFY_OK;
-+}
-+
-+void __init page_alloc_init(void)
-+{
-+ hotcpu_notifier(page_alloc_cpu_notify, 0);
-+}
-+
-+/*
-+ * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
-+ * or min_free_kbytes changes.
-+ */
-+static void calculate_totalreserve_pages(void)
-+{
-+ struct pglist_data *pgdat;
-+ unsigned long reserve_pages = 0;
-+ enum zone_type i, j;
-+
-+ for_each_online_pgdat(pgdat) {
-+ for (i = 0; i < MAX_NR_ZONES; i++) {
-+ struct zone *zone = pgdat->node_zones + i;
-+ long max = 0;
-+
-+ /* Find valid and maximum lowmem_reserve in the zone */
-+ for (j = i; j < MAX_NR_ZONES; j++) {
-+ if (zone->lowmem_reserve[j] > max)
-+ max = zone->lowmem_reserve[j];
-+ }
-+
-+ /* we treat the high watermark as reserved pages. */
-+ max += high_wmark_pages(zone);
-+
-+ if (max > zone->managed_pages)
-+ max = zone->managed_pages;
-+ reserve_pages += max;
-+ /*
-+ * Lowmem reserves are not available to
-+ * GFP_HIGHUSER page cache allocations and
-+ * kswapd tries to balance zones to their high
-+ * watermark. As a result, neither should be
-+ * regarded as dirtyable memory, to prevent a
-+ * situation where reclaim has to clean pages
-+ * in order to balance the zones.
-+ */
-+ zone->dirty_balance_reserve = max;
-+ }
-+ }
-+ dirty_balance_reserve = reserve_pages;
-+ totalreserve_pages = reserve_pages;
-+}
-+
-+/*
-+ * setup_per_zone_lowmem_reserve - called whenever
-+ * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone
-+ * has a correct pages reserved value, so an adequate number of
-+ * pages are left in the zone after a successful __alloc_pages().
-+ */
-+static void setup_per_zone_lowmem_reserve(void)
-+{
-+ struct pglist_data *pgdat;
-+ enum zone_type j, idx;
-+
-+ for_each_online_pgdat(pgdat) {
-+ for (j = 0; j < MAX_NR_ZONES; j++) {
-+ struct zone *zone = pgdat->node_zones + j;
-+ unsigned long managed_pages = zone->managed_pages;
-+
-+ zone->lowmem_reserve[j] = 0;
-+
-+ idx = j;
-+ while (idx) {
-+ struct zone *lower_zone;
-+
-+ idx--;
-+
-+ if (sysctl_lowmem_reserve_ratio[idx] < 1)
-+ sysctl_lowmem_reserve_ratio[idx] = 1;
-+
-+ lower_zone = pgdat->node_zones + idx;
-+ lower_zone->lowmem_reserve[j] = managed_pages /
-+ sysctl_lowmem_reserve_ratio[idx];
-+ managed_pages += lower_zone->managed_pages;
-+ }
-+ }
-+ }
-+
-+ /* update totalreserve_pages */
-+ calculate_totalreserve_pages();
-+}
-+
-+static void __setup_per_zone_wmarks(void)
-+{
-+ unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
-+ unsigned long lowmem_pages = 0;
-+ struct zone *zone;
-+ unsigned long flags;
-+
-+ /* Calculate total number of !ZONE_HIGHMEM pages */
-+ for_each_zone(zone) {
-+ if (!is_highmem(zone))
-+ lowmem_pages += zone->managed_pages;
-+ }
-+
-+ for_each_zone(zone) {
-+ u64 tmp;
-+
-+ spin_lock_irqsave(&zone->lock, flags);
-+ tmp = (u64)pages_min * zone->managed_pages;
-+ do_div(tmp, lowmem_pages);
-+ if (is_highmem(zone)) {
-+ /*
-+ * __GFP_HIGH and PF_MEMALLOC allocations usually don't
-+ * need highmem pages, so cap pages_min to a small
-+ * value here.
-+ *
-+ * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
-+ * deltas control asynch page reclaim, and so should
-+ * not be capped for highmem.
-+ */
-+ unsigned long min_pages;
-+
-+ min_pages = zone->managed_pages / 1024;
-+ min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
-+ zone->watermark[WMARK_MIN] = min_pages;
-+ } else {
-+ /*
-+ * If it's a lowmem zone, reserve a number of pages
-+ * proportionate to the zone's size.
-+ */
-+ zone->watermark[WMARK_MIN] = tmp;
-+ }
-+
-+ zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
-+ zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
-+
-+ __mod_zone_page_state(zone, NR_ALLOC_BATCH,
-+ high_wmark_pages(zone) - low_wmark_pages(zone) -
-+ atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]));
-+
-+ setup_zone_migrate_reserve(zone);
-+ spin_unlock_irqrestore(&zone->lock, flags);
-+ }
-+
-+ /* update totalreserve_pages */
-+ calculate_totalreserve_pages();
-+}
-+
-+/**
-+ * setup_per_zone_wmarks - called when min_free_kbytes changes
-+ * or when memory is hot-{added|removed}
-+ *
-+ * Ensures that the watermark[min,low,high] values for each zone are set
-+ * correctly with respect to min_free_kbytes.
-+ */
-+void setup_per_zone_wmarks(void)
-+{
-+ mutex_lock(&zonelists_mutex);
-+ __setup_per_zone_wmarks();
-+ mutex_unlock(&zonelists_mutex);
-+}
-+
-+/*
-+ * The inactive anon list should be small enough that the VM never has to
-+ * do too much work, but large enough that each inactive page has a chance
-+ * to be referenced again before it is swapped out.
-+ *
-+ * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
-+ * INACTIVE_ANON pages on this zone's LRU, maintained by the
-+ * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
-+ * the anonymous pages are kept on the inactive list.
-+ *
-+ * total target max
-+ * memory ratio inactive anon
-+ * -------------------------------------
-+ * 10MB 1 5MB
-+ * 100MB 1 50MB
-+ * 1GB 3 250MB
-+ * 10GB 10 0.9GB
-+ * 100GB 31 3GB
-+ * 1TB 101 10GB
-+ * 10TB 320 32GB
-+ */
-+static void __meminit calculate_zone_inactive_ratio(struct zone *zone)
-+{
-+ unsigned int gb, ratio;
-+
-+ /* Zone size in gigabytes */
-+ gb = zone->managed_pages >> (30 - PAGE_SHIFT);
-+ if (gb)
-+ ratio = int_sqrt(10 * gb);
-+ else
-+ ratio = 1;
-+
-+ zone->inactive_ratio = ratio;
-+}
-+
-+static void __meminit setup_per_zone_inactive_ratio(void)
-+{
-+ struct zone *zone;
-+
-+ for_each_zone(zone)
-+ calculate_zone_inactive_ratio(zone);
-+}
-+
-+/*
-+ * Initialise min_free_kbytes.
-+ *
-+ * For small machines we want it small (128k min). For large machines
-+ * we want it large (64MB max). But it is not linear, because network
-+ * bandwidth does not increase linearly with machine size. We use
-+ *
-+ * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
-+ * min_free_kbytes = sqrt(lowmem_kbytes * 16)
-+ *
-+ * which yields
-+ *
-+ * 16MB: 512k
-+ * 32MB: 724k
-+ * 64MB: 1024k
-+ * 128MB: 1448k
-+ * 256MB: 2048k
-+ * 512MB: 2896k
-+ * 1024MB: 4096k
-+ * 2048MB: 5792k
-+ * 4096MB: 8192k
-+ * 8192MB: 11584k
-+ * 16384MB: 16384k
-+ */
-+int __meminit init_per_zone_wmark_min(void)
-+{
-+ unsigned long lowmem_kbytes;
-+ int new_min_free_kbytes;
-+
-+ lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
-+ new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
-+
-+ if (new_min_free_kbytes > user_min_free_kbytes) {
-+ min_free_kbytes = new_min_free_kbytes;
-+ if (min_free_kbytes < 128)
-+ min_free_kbytes = 128;
-+ if (min_free_kbytes > 65536)
-+ min_free_kbytes = 65536;
-+ } else {
-+ pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
-+ new_min_free_kbytes, user_min_free_kbytes);
-+ }
-+ setup_per_zone_wmarks();
-+ refresh_zone_stat_thresholds();
-+ setup_per_zone_lowmem_reserve();
-+ setup_per_zone_inactive_ratio();
-+ return 0;
-+}
-+module_init(init_per_zone_wmark_min)
-+
-+/*
-+ * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
-+ * that we can call two helper functions whenever min_free_kbytes
-+ * changes.
-+ */
-+int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
-+ void __user *buffer, size_t *length, loff_t *ppos)
-+{
-+ int rc;
-+
-+ rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
-+ if (rc)
-+ return rc;
-+
-+ if (write) {
-+ user_min_free_kbytes = min_free_kbytes;
-+ setup_per_zone_wmarks();
-+ }
-+ return 0;
-+}
-+
-+#ifdef CONFIG_NUMA
-+int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
-+ void __user *buffer, size_t *length, loff_t *ppos)
-+{
-+ struct zone *zone;
-+ int rc;
-+
-+ rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
-+ if (rc)
-+ return rc;
-+
-+ for_each_zone(zone)
-+ zone->min_unmapped_pages = (zone->managed_pages *
-+ sysctl_min_unmapped_ratio) / 100;
-+ return 0;
-+}
-+
-+int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
-+ void __user *buffer, size_t *length, loff_t *ppos)
-+{
-+ struct zone *zone;
-+ int rc;
-+
-+ rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
-+ if (rc)
-+ return rc;
-+
-+ for_each_zone(zone)
-+ zone->min_slab_pages = (zone->managed_pages *
-+ sysctl_min_slab_ratio) / 100;
-+ return 0;
-+}
-+#endif
-+
-+/*
-+ * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
-+ * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
-+ * whenever sysctl_lowmem_reserve_ratio changes.
-+ *
-+ * The reserve ratio obviously has absolutely no relation with the
-+ * minimum watermarks. The lowmem reserve ratio can only make sense
-+ * if in function of the boot time zone sizes.
-+ */
-+int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
-+ void __user *buffer, size_t *length, loff_t *ppos)
-+{
-+ proc_dointvec_minmax(table, write, buffer, length, ppos);
-+ setup_per_zone_lowmem_reserve();
-+ return 0;
-+}
-+
-+/*
-+ * percpu_pagelist_fraction - changes the pcp->high for each zone on each
-+ * cpu. It is the fraction of total pages in each zone that a hot per cpu
-+ * pagelist can have before it gets flushed back to buddy allocator.
-+ */
-+int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
-+ void __user *buffer, size_t *length, loff_t *ppos)
-+{
-+ struct zone *zone;
-+ int old_percpu_pagelist_fraction;
-+ int ret;
-+
-+ mutex_lock(&pcp_batch_high_lock);
-+ old_percpu_pagelist_fraction = percpu_pagelist_fraction;
-+
-+ ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
-+ if (!write || ret < 0)
-+ goto out;
-+
-+ /* Sanity checking to avoid pcp imbalance */
-+ if (percpu_pagelist_fraction &&
-+ percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
-+ percpu_pagelist_fraction = old_percpu_pagelist_fraction;
-+ ret = -EINVAL;
-+ goto out;
-+ }
-+
-+ /* No change? */
-+ if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
-+ goto out;
-+
-+ for_each_populated_zone(zone) {
-+ unsigned int cpu;
-+
-+ for_each_possible_cpu(cpu)
-+ pageset_set_high_and_batch(zone,
-+ per_cpu_ptr(zone->pageset, cpu));
-+ }
-+out:
-+ mutex_unlock(&pcp_batch_high_lock);
-+ return ret;
-+}
-+
-+int hashdist = HASHDIST_DEFAULT;
-+
-+#ifdef CONFIG_NUMA
-+static int __init set_hashdist(char *str)
-+{
-+ if (!str)
-+ return 0;
-+ hashdist = simple_strtoul(str, &str, 0);
-+ return 1;
-+}
-+__setup("hashdist=", set_hashdist);
-+#endif
-+
-+/*
-+ * allocate a large system hash table from bootmem
-+ * - it is assumed that the hash table must contain an exact power-of-2
-+ * quantity of entries
-+ * - limit is the number of hash buckets, not the total allocation size
-+ */
-+void *__init alloc_large_system_hash(const char *tablename,
-+ unsigned long bucketsize,
-+ unsigned long numentries,
-+ int scale,
-+ int flags,
-+ unsigned int *_hash_shift,
-+ unsigned int *_hash_mask,
-+ unsigned long low_limit,
-+ unsigned long high_limit)
-+{
-+ unsigned long long max = high_limit;
-+ unsigned long log2qty, size;
-+ void *table = NULL;
-+
-+ /* allow the kernel cmdline to have a say */
-+ if (!numentries) {
-+ /* round applicable memory size up to nearest megabyte */
-+ numentries = nr_kernel_pages;
-+
-+ /* It isn't necessary when PAGE_SIZE >= 1MB */
-+ if (PAGE_SHIFT < 20)
-+ numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
-+
-+ /* limit to 1 bucket per 2^scale bytes of low memory */
-+ if (scale > PAGE_SHIFT)
-+ numentries >>= (scale - PAGE_SHIFT);
-+ else
-+ numentries <<= (PAGE_SHIFT - scale);
-+
-+ /* Make sure we've got at least a 0-order allocation.. */
-+ if (unlikely(flags & HASH_SMALL)) {
-+ /* Makes no sense without HASH_EARLY */
-+ WARN_ON(!(flags & HASH_EARLY));
-+ if (!(numentries >> *_hash_shift)) {
-+ numentries = 1UL << *_hash_shift;
-+ BUG_ON(!numentries);
-+ }
-+ } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
-+ numentries = PAGE_SIZE / bucketsize;
-+ }
-+ numentries = roundup_pow_of_two(numentries);
-+
-+ /* limit allocation size to 1/16 total memory by default */
-+ if (max == 0) {
-+ max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
-+ do_div(max, bucketsize);
-+ }
-+ max = min(max, 0x80000000ULL);
-+
-+ if (numentries < low_limit)
-+ numentries = low_limit;
-+ if (numentries > max)
-+ numentries = max;
-+
-+ log2qty = ilog2(numentries);
-+
-+ do {
-+ size = bucketsize << log2qty;
-+ if (flags & HASH_EARLY)
-+ table = memblock_virt_alloc_nopanic(size, 0);
-+ else if (hashdist)
-+ table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
-+ else {
-+ /*
-+ * If bucketsize is not a power-of-two, we may free
-+ * some pages at the end of hash table which
-+ * alloc_pages_exact() automatically does
-+ */
-+ if (get_order(size) < MAX_ORDER) {
-+ table = alloc_pages_exact(size, GFP_ATOMIC);
-+ kmemleak_alloc(table, size, 1, GFP_ATOMIC);
-+ }
-+ }
-+ } while (!table && size > PAGE_SIZE && --log2qty);
-+
-+ if (!table)
-+ panic("Failed to allocate %s hash table\n", tablename);
-+
-+ printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n",
-+ tablename,
-+ (1UL << log2qty),
-+ ilog2(size) - PAGE_SHIFT,
-+ size);
-+
-+ if (_hash_shift)
-+ *_hash_shift = log2qty;
-+ if (_hash_mask)
-+ *_hash_mask = (1 << log2qty) - 1;
-+
-+ return table;
-+}
-+
-+/* Return a pointer to the bitmap storing bits affecting a block of pages */
-+static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
-+ unsigned long pfn)
-+{
-+#ifdef CONFIG_SPARSEMEM
-+ return __pfn_to_section(pfn)->pageblock_flags;
-+#else
-+ return zone->pageblock_flags;
-+#endif /* CONFIG_SPARSEMEM */
-+}
-+
-+static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
-+{
-+#ifdef CONFIG_SPARSEMEM
-+ pfn &= (PAGES_PER_SECTION-1);
-+ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
-+#else
-+ pfn = pfn - round_down(zone->zone_start_pfn, pageblock_nr_pages);
-+ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
-+#endif /* CONFIG_SPARSEMEM */
-+}
-+
-+/**
-+ * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
-+ * @page: The page within the block of interest
-+ * @pfn: The target page frame number
-+ * @end_bitidx: The last bit of interest to retrieve
-+ * @mask: mask of bits that the caller is interested in
-+ *
-+ * Return: pageblock_bits flags
-+ */
-+unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
-+ unsigned long end_bitidx,
-+ unsigned long mask)
-+{
-+ struct zone *zone;
-+ unsigned long *bitmap;
-+ unsigned long bitidx, word_bitidx;
-+ unsigned long word;
-+
-+ zone = page_zone(page);
-+ bitmap = get_pageblock_bitmap(zone, pfn);
-+ bitidx = pfn_to_bitidx(zone, pfn);
-+ word_bitidx = bitidx / BITS_PER_LONG;
-+ bitidx &= (BITS_PER_LONG-1);
-+
-+ word = bitmap[word_bitidx];
-+ bitidx += end_bitidx;
-+ return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
-+}
-+
-+/**
-+ * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
-+ * @page: The page within the block of interest
-+ * @flags: The flags to set
-+ * @pfn: The target page frame number
-+ * @end_bitidx: The last bit of interest
-+ * @mask: mask of bits that the caller is interested in
-+ */
-+void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
-+ unsigned long pfn,
-+ unsigned long end_bitidx,
-+ unsigned long mask)
-+{
-+ struct zone *zone;
-+ unsigned long *bitmap;
-+ unsigned long bitidx, word_bitidx;
-+ unsigned long old_word, word;
-+
-+ BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
-+
-+ zone = page_zone(page);
-+ bitmap = get_pageblock_bitmap(zone, pfn);
-+ bitidx = pfn_to_bitidx(zone, pfn);
-+ word_bitidx = bitidx / BITS_PER_LONG;
-+ bitidx &= (BITS_PER_LONG-1);
-+
-+ VM_BUG_ON_PAGE(!zone_spans_pfn(zone, pfn), page);
-+
-+ bitidx += end_bitidx;
-+ mask <<= (BITS_PER_LONG - bitidx - 1);
-+ flags <<= (BITS_PER_LONG - bitidx - 1);
-+
-+ word = READ_ONCE(bitmap[word_bitidx]);
-+ for (;;) {
-+ old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
-+ if (word == old_word)
-+ break;
-+ word = old_word;
-+ }
-+}
-+
-+/*
-+ * This function checks whether pageblock includes unmovable pages or not.
-+ * If @count is not zero, it is okay to include less @count unmovable pages
-+ *
-+ * PageLRU check without isolation or lru_lock could race so that
-+ * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
-+ * expect this function should be exact.
-+ */
-+bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
-+ bool skip_hwpoisoned_pages)
-+{
-+ unsigned long pfn, iter, found;
-+ int mt;
-+
-+ /*
-+ * For avoiding noise data, lru_add_drain_all() should be called
-+ * If ZONE_MOVABLE, the zone never contains unmovable pages
-+ */
-+ if (zone_idx(zone) == ZONE_MOVABLE)
-+ return false;
-+ mt = get_pageblock_migratetype(page);
-+ if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
-+ return false;
-+
-+ pfn = page_to_pfn(page);
-+ for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
-+ unsigned long check = pfn + iter;
-+
-+ if (!pfn_valid_within(check))
-+ continue;
-+
-+ page = pfn_to_page(check);
-+
-+ /*
-+ * Hugepages are not in LRU lists, but they're movable.
-+ * We need not scan over tail pages bacause we don't
-+ * handle each tail page individually in migration.
-+ */
-+ if (PageHuge(page)) {
-+ iter = round_up(iter + 1, 1<<compound_order(page)) - 1;
-+ continue;
-+ }
-+
-+ /*
-+ * We can't use page_count without pin a page
-+ * because another CPU can free compound page.
-+ * This check already skips compound tails of THP
-+ * because their page->_count is zero at all time.
-+ */
-+ if (!atomic_read(&page->_count)) {
-+ if (PageBuddy(page))
-+ iter += (1 << page_order(page)) - 1;
-+ continue;
-+ }
-+
-+ /*
-+ * The HWPoisoned page may be not in buddy system, and
-+ * page_count() is not 0.
-+ */
-+ if (skip_hwpoisoned_pages && PageHWPoison(page))
-+ continue;
-+
-+ if (!PageLRU(page))
-+ found++;
-+ /*
-+ * If there are RECLAIMABLE pages, we need to check
-+ * it. But now, memory offline itself doesn't call
-+ * shrink_node_slabs() and it still to be fixed.
-+ */
-+ /*
-+ * If the page is not RAM, page_count()should be 0.
-+ * we don't need more check. This is an _used_ not-movable page.
-+ *
-+ * The problematic thing here is PG_reserved pages. PG_reserved
-+ * is set to both of a memory hole page and a _used_ kernel
-+ * page at boot.
-+ */
-+ if (found > count)
-+ return true;
-+ }
-+ return false;
-+}
-+
-+bool is_pageblock_removable_nolock(struct page *page)
-+{
-+ struct zone *zone;
-+ unsigned long pfn;
-+
-+ /*
-+ * We have to be careful here because we are iterating over memory
-+ * sections which are not zone aware so we might end up outside of
-+ * the zone but still within the section.
-+ * We have to take care about the node as well. If the node is offline
-+ * its NODE_DATA will be NULL - see page_zone.
-+ */
-+ if (!node_online(page_to_nid(page)))
-+ return false;
-+
-+ zone = page_zone(page);
-+ pfn = page_to_pfn(page);
-+ if (!zone_spans_pfn(zone, pfn))
-+ return false;
-+
-+ return !has_unmovable_pages(zone, page, 0, true);
-+}
-+
-+#ifdef CONFIG_CMA
-+
-+static unsigned long pfn_max_align_down(unsigned long pfn)
-+{
-+ return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
-+ pageblock_nr_pages) - 1);
-+}
-+
-+static unsigned long pfn_max_align_up(unsigned long pfn)
-+{
-+ return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
-+ pageblock_nr_pages));
-+}
-+
-+/* [start, end) must belong to a single zone. */
-+static int __alloc_contig_migrate_range(struct compact_control *cc,
-+ unsigned long start, unsigned long end)
-+{
-+ /* This function is based on compact_zone() from compaction.c. */
-+ unsigned long nr_reclaimed;
-+ unsigned long pfn = start;
-+ unsigned int tries = 0;
-+ int ret = 0;
-+
-+ migrate_prep();
-+
-+ while (pfn < end || !list_empty(&cc->migratepages)) {
-+ if (fatal_signal_pending(current)) {
-+ ret = -EINTR;
-+ break;
-+ }
-+
-+ if (list_empty(&cc->migratepages)) {
-+ cc->nr_migratepages = 0;
-+ pfn = isolate_migratepages_range(cc, pfn, end);
-+ if (!pfn) {
-+ ret = -EINTR;
-+ break;
-+ }
-+ tries = 0;
-+ } else if (++tries == 5) {
-+ ret = ret < 0 ? ret : -EBUSY;
-+ break;
-+ }
-+
-+ nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
-+ &cc->migratepages);
-+ cc->nr_migratepages -= nr_reclaimed;
-+
-+ ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
-+ NULL, 0, cc->mode, MR_CMA);
-+ }
-+ if (ret < 0) {
-+ putback_movable_pages(&cc->migratepages);
-+ return ret;
-+ }
-+ return 0;
-+}
-+
-+/**
-+ * alloc_contig_range() -- tries to allocate given range of pages
-+ * @start: start PFN to allocate
-+ * @end: one-past-the-last PFN to allocate
-+ * @migratetype: migratetype of the underlaying pageblocks (either
-+ * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
-+ * in range must have the same migratetype and it must
-+ * be either of the two.
-+ *
-+ * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
-+ * aligned, however it's the caller's responsibility to guarantee that
-+ * we are the only thread that changes migrate type of pageblocks the
-+ * pages fall in.
-+ *
-+ * The PFN range must belong to a single zone.
-+ *
-+ * Returns zero on success or negative error code. On success all
-+ * pages which PFN is in [start, end) are allocated for the caller and
-+ * need to be freed with free_contig_range().
-+ */
-+int alloc_contig_range(unsigned long start, unsigned long end,
-+ unsigned migratetype)
-+{
-+ unsigned long outer_start, outer_end;
-+ int ret = 0, order;
-+
-+ struct compact_control cc = {
-+ .nr_migratepages = 0,
-+ .order = -1,
-+ .zone = page_zone(pfn_to_page(start)),
-+ .mode = MIGRATE_SYNC,
-+ .ignore_skip_hint = true,
-+ };
-+ INIT_LIST_HEAD(&cc.migratepages);
-+
-+ /*
-+ * What we do here is we mark all pageblocks in range as
-+ * MIGRATE_ISOLATE. Because pageblock and max order pages may
-+ * have different sizes, and due to the way page allocator
-+ * work, we align the range to biggest of the two pages so
-+ * that page allocator won't try to merge buddies from
-+ * different pageblocks and change MIGRATE_ISOLATE to some
-+ * other migration type.
-+ *
-+ * Once the pageblocks are marked as MIGRATE_ISOLATE, we
-+ * migrate the pages from an unaligned range (ie. pages that
-+ * we are interested in). This will put all the pages in
-+ * range back to page allocator as MIGRATE_ISOLATE.
-+ *
-+ * When this is done, we take the pages in range from page
-+ * allocator removing them from the buddy system. This way
-+ * page allocator will never consider using them.
-+ *
-+ * This lets us mark the pageblocks back as
-+ * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
-+ * aligned range but not in the unaligned, original range are
-+ * put back to page allocator so that buddy can use them.
-+ */
-+
-+ ret = start_isolate_page_range(pfn_max_align_down(start),
-+ pfn_max_align_up(end), migratetype,
-+ false);
-+ if (ret)
-+ return ret;
-+
-+ ret = __alloc_contig_migrate_range(&cc, start, end);
-+ if (ret)
-+ goto done;
-+
-+ /*
-+ * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
-+ * aligned blocks that are marked as MIGRATE_ISOLATE. What's
-+ * more, all pages in [start, end) are free in page allocator.
-+ * What we are going to do is to allocate all pages from
-+ * [start, end) (that is remove them from page allocator).
-+ *
-+ * The only problem is that pages at the beginning and at the
-+ * end of interesting range may be not aligned with pages that
-+ * page allocator holds, ie. they can be part of higher order
-+ * pages. Because of this, we reserve the bigger range and
-+ * once this is done free the pages we are not interested in.
-+ *
-+ * We don't have to hold zone->lock here because the pages are
-+ * isolated thus they won't get removed from buddy.
-+ */
-+
-+ lru_add_drain_all();
-+ drain_all_pages(cc.zone);
-+
-+ order = 0;
-+ outer_start = start;
-+ while (!PageBuddy(pfn_to_page(outer_start))) {
-+ if (++order >= MAX_ORDER) {
-+ ret = -EBUSY;
-+ goto done;
-+ }
-+ outer_start &= ~0UL << order;
-+ }
-+
-+ /* Make sure the range is really isolated. */
-+ if (test_pages_isolated(outer_start, end, false)) {
-+ pr_info("%s: [%lx, %lx) PFNs busy\n",
-+ __func__, outer_start, end);
-+ ret = -EBUSY;
-+ goto done;
-+ }
-+
-+ /* Grab isolated pages from freelists. */
-+ outer_end = isolate_freepages_range(&cc, outer_start, end);
-+ if (!outer_end) {
-+ ret = -EBUSY;
-+ goto done;
-+ }
-+
-+ /* Free head and tail (if any) */
-+ if (start != outer_start)
-+ free_contig_range(outer_start, start - outer_start);
-+ if (end != outer_end)
-+ free_contig_range(end, outer_end - end);
-+
-+done:
-+ undo_isolate_page_range(pfn_max_align_down(start),
-+ pfn_max_align_up(end), migratetype);
-+ return ret;
-+}
-+
-+void free_contig_range(unsigned long pfn, unsigned nr_pages)
-+{
-+ unsigned int count = 0;
-+
-+ for (; nr_pages--; pfn++) {
-+ struct page *page = pfn_to_page(pfn);
-+
-+ count += page_count(page) != 1;
-+ __free_page(page);
-+ }
-+ WARN(count != 0, "%d pages are still in use!\n", count);
-+}
-+#endif
-+
-+#ifdef CONFIG_MEMORY_HOTPLUG
-+/*
-+ * The zone indicated has a new number of managed_pages; batch sizes and percpu
-+ * page high values need to be recalulated.
-+ */
-+void __meminit zone_pcp_update(struct zone *zone)
-+{
-+ unsigned cpu;
-+ mutex_lock(&pcp_batch_high_lock);
-+ for_each_possible_cpu(cpu)
-+ pageset_set_high_and_batch(zone,
-+ per_cpu_ptr(zone->pageset, cpu));
-+ mutex_unlock(&pcp_batch_high_lock);
-+}
-+#endif
-+
-+void zone_pcp_reset(struct zone *zone)
-+{
-+ unsigned long flags;
-+ int cpu;
-+ struct per_cpu_pageset *pset;
-+
-+ /* avoid races with drain_pages() */
-+ local_irq_save(flags);
-+ if (zone->pageset != &boot_pageset) {
-+ for_each_online_cpu(cpu) {
-+ pset = per_cpu_ptr(zone->pageset, cpu);
-+ drain_zonestat(zone, pset);
-+ }
-+ free_percpu(zone->pageset);
-+ zone->pageset = &boot_pageset;
-+ }
-+ local_irq_restore(flags);
-+}
-+
-+#ifdef CONFIG_MEMORY_HOTREMOVE
-+/*
-+ * All pages in the range must be isolated before calling this.
-+ */
-+void
-+__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
-+{
-+ struct page *page;
-+ struct zone *zone;
-+ unsigned int order, i;
-+ unsigned long pfn;
-+ unsigned long flags;
-+ /* find the first valid pfn */
-+ for (pfn = start_pfn; pfn < end_pfn; pfn++)
-+ if (pfn_valid(pfn))
-+ break;
-+ if (pfn == end_pfn)
-+ return;
-+ zone = page_zone(pfn_to_page(pfn));
-+ spin_lock_irqsave(&zone->lock, flags);
-+ pfn = start_pfn;
-+ while (pfn < end_pfn) {
-+ if (!pfn_valid(pfn)) {
-+ pfn++;
-+ continue;
-+ }
-+ page = pfn_to_page(pfn);
-+ /*
-+ * The HWPoisoned page may be not in buddy system, and
-+ * page_count() is not 0.
-+ */
-+ if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
-+ pfn++;
-+ SetPageReserved(page);
-+ continue;
-+ }
-+
-+ BUG_ON(page_count(page));
-+ BUG_ON(!PageBuddy(page));
-+ order = page_order(page);
-+#ifdef CONFIG_DEBUG_VM
-+ printk(KERN_INFO "remove from free list %lx %d %lx\n",
-+ pfn, 1 << order, end_pfn);
-+#endif
-+ list_del(&page->lru);
-+ rmv_page_order(page);
-+ zone->free_area[order].nr_free--;
-+ for (i = 0; i < (1 << order); i++)
-+ SetPageReserved((page+i));
-+ pfn += (1 << order);
-+ }
-+ spin_unlock_irqrestore(&zone->lock, flags);
-+}
-+#endif
-+
-+#ifdef CONFIG_MEMORY_FAILURE
-+bool is_free_buddy_page(struct page *page)
-+{
-+ struct zone *zone = page_zone(page);
-+ unsigned long pfn = page_to_pfn(page);
-+ unsigned long flags;
-+ unsigned int order;
-+
-+ spin_lock_irqsave(&zone->lock, flags);
-+ for (order = 0; order < MAX_ORDER; order++) {
-+ struct page *page_head = page - (pfn & ((1 << order) - 1));
-+
-+ if (PageBuddy(page_head) && page_order(page_head) >= order)
-+ break;
-+ }
-+ spin_unlock_irqrestore(&zone->lock, flags);
-+
-+ return order < MAX_ORDER;
-+}
-+#endif
diff -Nur linux-4.1.10.orig/mm/slab.h linux-4.1.10/mm/slab.h
--- linux-4.1.10.orig/mm/slab.h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/slab.h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/slab.h 2015-10-12 22:33:32.332673054 +0200
@@ -330,7 +330,11 @@
* The slab lists for all objects.
*/
@@ -56091,7 +25652,7 @@ diff -Nur linux-4.1.10.orig/mm/slab.h linux-4.1.10/mm/slab.h
struct list_head slabs_partial; /* partial list first, better asm code */
diff -Nur linux-4.1.10.orig/mm/slub.c linux-4.1.10/mm/slub.c
--- linux-4.1.10.orig/mm/slub.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/slub.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/slub.c 2015-10-12 22:33:32.332673054 +0200
@@ -1069,7 +1069,7 @@
{
struct kmem_cache_node *n = get_node(s, page_to_nid(page));
@@ -56576,5357 +26137,9 @@ diff -Nur linux-4.1.10.orig/mm/slub.c linux-4.1.10/mm/slub.c
}
for (i = 0; i < t.count; i++) {
-diff -Nur linux-4.1.10.orig/mm/slub.c.orig linux-4.1.10/mm/slub.c.orig
---- linux-4.1.10.orig/mm/slub.c.orig 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/mm/slub.c.orig 2015-10-03 13:49:38.000000000 +0200
-@@ -0,0 +1,5344 @@
-+/*
-+ * SLUB: A slab allocator that limits cache line use instead of queuing
-+ * objects in per cpu and per node lists.
-+ *
-+ * The allocator synchronizes using per slab locks or atomic operatios
-+ * and only uses a centralized lock to manage a pool of partial slabs.
-+ *
-+ * (C) 2007 SGI, Christoph Lameter
-+ * (C) 2011 Linux Foundation, Christoph Lameter
-+ */
-+
-+#include <linux/mm.h>
-+#include <linux/swap.h> /* struct reclaim_state */
-+#include <linux/module.h>
-+#include <linux/bit_spinlock.h>
-+#include <linux/interrupt.h>
-+#include <linux/bitops.h>
-+#include <linux/slab.h>
-+#include "slab.h"
-+#include <linux/proc_fs.h>
-+#include <linux/notifier.h>
-+#include <linux/seq_file.h>
-+#include <linux/kasan.h>
-+#include <linux/kmemcheck.h>
-+#include <linux/cpu.h>
-+#include <linux/cpuset.h>
-+#include <linux/mempolicy.h>
-+#include <linux/ctype.h>
-+#include <linux/debugobjects.h>
-+#include <linux/kallsyms.h>
-+#include <linux/memory.h>
-+#include <linux/math64.h>
-+#include <linux/fault-inject.h>
-+#include <linux/stacktrace.h>
-+#include <linux/prefetch.h>
-+#include <linux/memcontrol.h>
-+
-+#include <trace/events/kmem.h>
-+
-+#include "internal.h"
-+
-+/*
-+ * Lock order:
-+ * 1. slab_mutex (Global Mutex)
-+ * 2. node->list_lock
-+ * 3. slab_lock(page) (Only on some arches and for debugging)
-+ *
-+ * slab_mutex
-+ *
-+ * The role of the slab_mutex is to protect the list of all the slabs
-+ * and to synchronize major metadata changes to slab cache structures.
-+ *
-+ * The slab_lock is only used for debugging and on arches that do not
-+ * have the ability to do a cmpxchg_double. It only protects the second
-+ * double word in the page struct. Meaning
-+ * A. page->freelist -> List of object free in a page
-+ * B. page->counters -> Counters of objects
-+ * C. page->frozen -> frozen state
-+ *
-+ * If a slab is frozen then it is exempt from list management. It is not
-+ * on any list. The processor that froze the slab is the one who can
-+ * perform list operations on the page. Other processors may put objects
-+ * onto the freelist but the processor that froze the slab is the only
-+ * one that can retrieve the objects from the page's freelist.
-+ *
-+ * The list_lock protects the partial and full list on each node and
-+ * the partial slab counter. If taken then no new slabs may be added or
-+ * removed from the lists nor make the number of partial slabs be modified.
-+ * (Note that the total number of slabs is an atomic value that may be
-+ * modified without taking the list lock).
-+ *
-+ * The list_lock is a centralized lock and thus we avoid taking it as
-+ * much as possible. As long as SLUB does not have to handle partial
-+ * slabs, operations can continue without any centralized lock. F.e.
-+ * allocating a long series of objects that fill up slabs does not require
-+ * the list lock.
-+ * Interrupts are disabled during allocation and deallocation in order to
-+ * make the slab allocator safe to use in the context of an irq. In addition
-+ * interrupts are disabled to ensure that the processor does not change
-+ * while handling per_cpu slabs, due to kernel preemption.
-+ *
-+ * SLUB assigns one slab for allocation to each processor.
-+ * Allocations only occur from these slabs called cpu slabs.
-+ *
-+ * Slabs with free elements are kept on a partial list and during regular
-+ * operations no list for full slabs is used. If an object in a full slab is
-+ * freed then the slab will show up again on the partial lists.
-+ * We track full slabs for debugging purposes though because otherwise we
-+ * cannot scan all objects.
-+ *
-+ * Slabs are freed when they become empty. Teardown and setup is
-+ * minimal so we rely on the page allocators per cpu caches for
-+ * fast frees and allocs.
-+ *
-+ * Overloading of page flags that are otherwise used for LRU management.
-+ *
-+ * PageActive The slab is frozen and exempt from list processing.
-+ * This means that the slab is dedicated to a purpose
-+ * such as satisfying allocations for a specific
-+ * processor. Objects may be freed in the slab while
-+ * it is frozen but slab_free will then skip the usual
-+ * list operations. It is up to the processor holding
-+ * the slab to integrate the slab into the slab lists
-+ * when the slab is no longer needed.
-+ *
-+ * One use of this flag is to mark slabs that are
-+ * used for allocations. Then such a slab becomes a cpu
-+ * slab. The cpu slab may be equipped with an additional
-+ * freelist that allows lockless access to
-+ * free objects in addition to the regular freelist
-+ * that requires the slab lock.
-+ *
-+ * PageError Slab requires special handling due to debug
-+ * options set. This moves slab handling out of
-+ * the fast path and disables lockless freelists.
-+ */
-+
-+static inline int kmem_cache_debug(struct kmem_cache *s)
-+{
-+#ifdef CONFIG_SLUB_DEBUG
-+ return unlikely(s->flags & SLAB_DEBUG_FLAGS);
-+#else
-+ return 0;
-+#endif
-+}
-+
-+static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
-+{
-+#ifdef CONFIG_SLUB_CPU_PARTIAL
-+ return !kmem_cache_debug(s);
-+#else
-+ return false;
-+#endif
-+}
-+
-+/*
-+ * Issues still to be resolved:
-+ *
-+ * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
-+ *
-+ * - Variable sizing of the per node arrays
-+ */
-+
-+/* Enable to test recovery from slab corruption on boot */
-+#undef SLUB_RESILIENCY_TEST
-+
-+/* Enable to log cmpxchg failures */
-+#undef SLUB_DEBUG_CMPXCHG
-+
-+/*
-+ * Mininum number of partial slabs. These will be left on the partial
-+ * lists even if they are empty. kmem_cache_shrink may reclaim them.
-+ */
-+#define MIN_PARTIAL 5
-+
-+/*
-+ * Maximum number of desirable partial slabs.
-+ * The existence of more partial slabs makes kmem_cache_shrink
-+ * sort the partial list by the number of objects in use.
-+ */
-+#define MAX_PARTIAL 10
-+
-+#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
-+ SLAB_POISON | SLAB_STORE_USER)
-+
-+/*
-+ * Debugging flags that require metadata to be stored in the slab. These get
-+ * disabled when slub_debug=O is used and a cache's min order increases with
-+ * metadata.
-+ */
-+#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
-+
-+#define OO_SHIFT 16
-+#define OO_MASK ((1 << OO_SHIFT) - 1)
-+#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
-+
-+/* Internal SLUB flags */
-+#define __OBJECT_POISON 0x80000000UL /* Poison object */
-+#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
-+
-+#ifdef CONFIG_SMP
-+static struct notifier_block slab_notifier;
-+#endif
-+
-+/*
-+ * Tracking user of a slab.
-+ */
-+#define TRACK_ADDRS_COUNT 16
-+struct track {
-+ unsigned long addr; /* Called from address */
-+#ifdef CONFIG_STACKTRACE
-+ unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
-+#endif
-+ int cpu; /* Was running on cpu */
-+ int pid; /* Pid context */
-+ unsigned long when; /* When did the operation occur */
-+};
-+
-+enum track_item { TRACK_ALLOC, TRACK_FREE };
-+
-+#ifdef CONFIG_SYSFS
-+static int sysfs_slab_add(struct kmem_cache *);
-+static int sysfs_slab_alias(struct kmem_cache *, const char *);
-+static void memcg_propagate_slab_attrs(struct kmem_cache *s);
-+#else
-+static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
-+static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
-+ { return 0; }
-+static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
-+#endif
-+
-+static inline void stat(const struct kmem_cache *s, enum stat_item si)
-+{
-+#ifdef CONFIG_SLUB_STATS
-+ /*
-+ * The rmw is racy on a preemptible kernel but this is acceptable, so
-+ * avoid this_cpu_add()'s irq-disable overhead.
-+ */
-+ raw_cpu_inc(s->cpu_slab->stat[si]);
-+#endif
-+}
-+
-+/********************************************************************
-+ * Core slab cache functions
-+ *******************************************************************/
-+
-+/* Verify that a pointer has an address that is valid within a slab page */
-+static inline int check_valid_pointer(struct kmem_cache *s,
-+ struct page *page, const void *object)
-+{
-+ void *base;
-+
-+ if (!object)
-+ return 1;
-+
-+ base = page_address(page);
-+ if (object < base || object >= base + page->objects * s->size ||
-+ (object - base) % s->size) {
-+ return 0;
-+ }
-+
-+ return 1;
-+}
-+
-+static inline void *get_freepointer(struct kmem_cache *s, void *object)
-+{
-+ return *(void **)(object + s->offset);
-+}
-+
-+static void prefetch_freepointer(const struct kmem_cache *s, void *object)
-+{
-+ prefetch(object + s->offset);
-+}
-+
-+static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
-+{
-+ void *p;
-+
-+#ifdef CONFIG_DEBUG_PAGEALLOC
-+ probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
-+#else
-+ p = get_freepointer(s, object);
-+#endif
-+ return p;
-+}
-+
-+static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
-+{
-+ *(void **)(object + s->offset) = fp;
-+}
-+
-+/* Loop over all objects in a slab */
-+#define for_each_object(__p, __s, __addr, __objects) \
-+ for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
-+ __p += (__s)->size)
-+
-+#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \
-+ for (__p = (__addr), __idx = 1; __idx <= __objects;\
-+ __p += (__s)->size, __idx++)
-+
-+/* Determine object index from a given position */
-+static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
-+{
-+ return (p - addr) / s->size;
-+}
-+
-+static inline size_t slab_ksize(const struct kmem_cache *s)
-+{
-+#ifdef CONFIG_SLUB_DEBUG
-+ /*
-+ * Debugging requires use of the padding between object
-+ * and whatever may come after it.
-+ */
-+ if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
-+ return s->object_size;
-+
-+#endif
-+ /*
-+ * If we have the need to store the freelist pointer
-+ * back there or track user information then we can
-+ * only use the space before that information.
-+ */
-+ if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
-+ return s->inuse;
-+ /*
-+ * Else we can use all the padding etc for the allocation
-+ */
-+ return s->size;
-+}
-+
-+static inline int order_objects(int order, unsigned long size, int reserved)
-+{
-+ return ((PAGE_SIZE << order) - reserved) / size;
-+}
-+
-+static inline struct kmem_cache_order_objects oo_make(int order,
-+ unsigned long size, int reserved)
-+{
-+ struct kmem_cache_order_objects x = {
-+ (order << OO_SHIFT) + order_objects(order, size, reserved)
-+ };
-+
-+ return x;
-+}
-+
-+static inline int oo_order(struct kmem_cache_order_objects x)
-+{
-+ return x.x >> OO_SHIFT;
-+}
-+
-+static inline int oo_objects(struct kmem_cache_order_objects x)
-+{
-+ return x.x & OO_MASK;
-+}
-+
-+/*
-+ * Per slab locking using the pagelock
-+ */
-+static __always_inline void slab_lock(struct page *page)
-+{
-+ bit_spin_lock(PG_locked, &page->flags);
-+}
-+
-+static __always_inline void slab_unlock(struct page *page)
-+{
-+ __bit_spin_unlock(PG_locked, &page->flags);
-+}
-+
-+static inline void set_page_slub_counters(struct page *page, unsigned long counters_new)
-+{
-+ struct page tmp;
-+ tmp.counters = counters_new;
-+ /*
-+ * page->counters can cover frozen/inuse/objects as well
-+ * as page->_count. If we assign to ->counters directly
-+ * we run the risk of losing updates to page->_count, so
-+ * be careful and only assign to the fields we need.
-+ */
-+ page->frozen = tmp.frozen;
-+ page->inuse = tmp.inuse;
-+ page->objects = tmp.objects;
-+}
-+
-+/* Interrupts must be disabled (for the fallback code to work right) */
-+static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
-+ void *freelist_old, unsigned long counters_old,
-+ void *freelist_new, unsigned long counters_new,
-+ const char *n)
-+{
-+ VM_BUG_ON(!irqs_disabled());
-+#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
-+ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
-+ if (s->flags & __CMPXCHG_DOUBLE) {
-+ if (cmpxchg_double(&page->freelist, &page->counters,
-+ freelist_old, counters_old,
-+ freelist_new, counters_new))
-+ return true;
-+ } else
-+#endif
-+ {
-+ slab_lock(page);
-+ if (page->freelist == freelist_old &&
-+ page->counters == counters_old) {
-+ page->freelist = freelist_new;
-+ set_page_slub_counters(page, counters_new);
-+ slab_unlock(page);
-+ return true;
-+ }
-+ slab_unlock(page);
-+ }
-+
-+ cpu_relax();
-+ stat(s, CMPXCHG_DOUBLE_FAIL);
-+
-+#ifdef SLUB_DEBUG_CMPXCHG
-+ pr_info("%s %s: cmpxchg double redo ", n, s->name);
-+#endif
-+
-+ return false;
-+}
-+
-+static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
-+ void *freelist_old, unsigned long counters_old,
-+ void *freelist_new, unsigned long counters_new,
-+ const char *n)
-+{
-+#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
-+ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
-+ if (s->flags & __CMPXCHG_DOUBLE) {
-+ if (cmpxchg_double(&page->freelist, &page->counters,
-+ freelist_old, counters_old,
-+ freelist_new, counters_new))
-+ return true;
-+ } else
-+#endif
-+ {
-+ unsigned long flags;
-+
-+ local_irq_save(flags);
-+ slab_lock(page);
-+ if (page->freelist == freelist_old &&
-+ page->counters == counters_old) {
-+ page->freelist = freelist_new;
-+ set_page_slub_counters(page, counters_new);
-+ slab_unlock(page);
-+ local_irq_restore(flags);
-+ return true;
-+ }
-+ slab_unlock(page);
-+ local_irq_restore(flags);
-+ }
-+
-+ cpu_relax();
-+ stat(s, CMPXCHG_DOUBLE_FAIL);
-+
-+#ifdef SLUB_DEBUG_CMPXCHG
-+ pr_info("%s %s: cmpxchg double redo ", n, s->name);
-+#endif
-+
-+ return false;
-+}
-+
-+#ifdef CONFIG_SLUB_DEBUG
-+/*
-+ * Determine a map of object in use on a page.
-+ *
-+ * Node listlock must be held to guarantee that the page does
-+ * not vanish from under us.
-+ */
-+static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
-+{
-+ void *p;
-+ void *addr = page_address(page);
-+
-+ for (p = page->freelist; p; p = get_freepointer(s, p))
-+ set_bit(slab_index(p, s, addr), map);
-+}
-+
-+/*
-+ * Debug settings:
-+ */
-+#ifdef CONFIG_SLUB_DEBUG_ON
-+static int slub_debug = DEBUG_DEFAULT_FLAGS;
-+#else
-+static int slub_debug;
-+#endif
-+
-+static char *slub_debug_slabs;
-+static int disable_higher_order_debug;
-+
-+/*
-+ * slub is about to manipulate internal object metadata. This memory lies
-+ * outside the range of the allocated object, so accessing it would normally
-+ * be reported by kasan as a bounds error. metadata_access_enable() is used
-+ * to tell kasan that these accesses are OK.
-+ */
-+static inline void metadata_access_enable(void)
-+{
-+ kasan_disable_current();
-+}
-+
-+static inline void metadata_access_disable(void)
-+{
-+ kasan_enable_current();
-+}
-+
-+/*
-+ * Object debugging
-+ */
-+static void print_section(char *text, u8 *addr, unsigned int length)
-+{
-+ metadata_access_enable();
-+ print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
-+ length, 1);
-+ metadata_access_disable();
-+}
-+
-+static struct track *get_track(struct kmem_cache *s, void *object,
-+ enum track_item alloc)
-+{
-+ struct track *p;
-+
-+ if (s->offset)
-+ p = object + s->offset + sizeof(void *);
-+ else
-+ p = object + s->inuse;
-+
-+ return p + alloc;
-+}
-+
-+static void set_track(struct kmem_cache *s, void *object,
-+ enum track_item alloc, unsigned long addr)
-+{
-+ struct track *p = get_track(s, object, alloc);
-+
-+ if (addr) {
-+#ifdef CONFIG_STACKTRACE
-+ struct stack_trace trace;
-+ int i;
-+
-+ trace.nr_entries = 0;
-+ trace.max_entries = TRACK_ADDRS_COUNT;
-+ trace.entries = p->addrs;
-+ trace.skip = 3;
-+ metadata_access_enable();
-+ save_stack_trace(&trace);
-+ metadata_access_disable();
-+
-+ /* See rant in lockdep.c */
-+ if (trace.nr_entries != 0 &&
-+ trace.entries[trace.nr_entries - 1] == ULONG_MAX)
-+ trace.nr_entries--;
-+
-+ for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
-+ p->addrs[i] = 0;
-+#endif
-+ p->addr = addr;
-+ p->cpu = smp_processor_id();
-+ p->pid = current->pid;
-+ p->when = jiffies;
-+ } else
-+ memset(p, 0, sizeof(struct track));
-+}
-+
-+static void init_tracking(struct kmem_cache *s, void *object)
-+{
-+ if (!(s->flags & SLAB_STORE_USER))
-+ return;
-+
-+ set_track(s, object, TRACK_FREE, 0UL);
-+ set_track(s, object, TRACK_ALLOC, 0UL);
-+}
-+
-+static void print_track(const char *s, struct track *t)
-+{
-+ if (!t->addr)
-+ return;
-+
-+ pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
-+ s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
-+#ifdef CONFIG_STACKTRACE
-+ {
-+ int i;
-+ for (i = 0; i < TRACK_ADDRS_COUNT; i++)
-+ if (t->addrs[i])
-+ pr_err("\t%pS\n", (void *)t->addrs[i]);
-+ else
-+ break;
-+ }
-+#endif
-+}
-+
-+static void print_tracking(struct kmem_cache *s, void *object)
-+{
-+ if (!(s->flags & SLAB_STORE_USER))
-+ return;
-+
-+ print_track("Allocated", get_track(s, object, TRACK_ALLOC));
-+ print_track("Freed", get_track(s, object, TRACK_FREE));
-+}
-+
-+static void print_page_info(struct page *page)
-+{
-+ pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
-+ page, page->objects, page->inuse, page->freelist, page->flags);
-+
-+}
-+
-+static void slab_bug(struct kmem_cache *s, char *fmt, ...)
-+{
-+ struct va_format vaf;
-+ va_list args;
-+
-+ va_start(args, fmt);
-+ vaf.fmt = fmt;
-+ vaf.va = &args;
-+ pr_err("=============================================================================\n");
-+ pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
-+ pr_err("-----------------------------------------------------------------------------\n\n");
-+
-+ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
-+ va_end(args);
-+}
-+
-+static void slab_fix(struct kmem_cache *s, char *fmt, ...)
-+{
-+ struct va_format vaf;
-+ va_list args;
-+
-+ va_start(args, fmt);
-+ vaf.fmt = fmt;
-+ vaf.va = &args;
-+ pr_err("FIX %s: %pV\n", s->name, &vaf);
-+ va_end(args);
-+}
-+
-+static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
-+{
-+ unsigned int off; /* Offset of last byte */
-+ u8 *addr = page_address(page);
-+
-+ print_tracking(s, p);
-+
-+ print_page_info(page);
-+
-+ pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
-+ p, p - addr, get_freepointer(s, p));
-+
-+ if (p > addr + 16)
-+ print_section("Bytes b4 ", p - 16, 16);
-+
-+ print_section("Object ", p, min_t(unsigned long, s->object_size,
-+ PAGE_SIZE));
-+ if (s->flags & SLAB_RED_ZONE)
-+ print_section("Redzone ", p + s->object_size,
-+ s->inuse - s->object_size);
-+
-+ if (s->offset)
-+ off = s->offset + sizeof(void *);
-+ else
-+ off = s->inuse;
-+
-+ if (s->flags & SLAB_STORE_USER)
-+ off += 2 * sizeof(struct track);
-+
-+ if (off != s->size)
-+ /* Beginning of the filler is the free pointer */
-+ print_section("Padding ", p + off, s->size - off);
-+
-+ dump_stack();
-+}
-+
-+void object_err(struct kmem_cache *s, struct page *page,
-+ u8 *object, char *reason)
-+{
-+ slab_bug(s, "%s", reason);
-+ print_trailer(s, page, object);
-+}
-+
-+static void slab_err(struct kmem_cache *s, struct page *page,
-+ const char *fmt, ...)
-+{
-+ va_list args;
-+ char buf[100];
-+
-+ va_start(args, fmt);
-+ vsnprintf(buf, sizeof(buf), fmt, args);
-+ va_end(args);
-+ slab_bug(s, "%s", buf);
-+ print_page_info(page);
-+ dump_stack();
-+}
-+
-+static void init_object(struct kmem_cache *s, void *object, u8 val)
-+{
-+ u8 *p = object;
-+
-+ if (s->flags & __OBJECT_POISON) {
-+ memset(p, POISON_FREE, s->object_size - 1);
-+ p[s->object_size - 1] = POISON_END;
-+ }
-+
-+ if (s->flags & SLAB_RED_ZONE)
-+ memset(p + s->object_size, val, s->inuse - s->object_size);
-+}
-+
-+static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
-+ void *from, void *to)
-+{
-+ slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
-+ memset(from, data, to - from);
-+}
-+
-+static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
-+ u8 *object, char *what,
-+ u8 *start, unsigned int value, unsigned int bytes)
-+{
-+ u8 *fault;
-+ u8 *end;
-+
-+ metadata_access_enable();
-+ fault = memchr_inv(start, value, bytes);
-+ metadata_access_disable();
-+ if (!fault)
-+ return 1;
-+
-+ end = start + bytes;
-+ while (end > fault && end[-1] == value)
-+ end--;
-+
-+ slab_bug(s, "%s overwritten", what);
-+ pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
-+ fault, end - 1, fault[0], value);
-+ print_trailer(s, page, object);
-+
-+ restore_bytes(s, what, value, fault, end);
-+ return 0;
-+}
-+
-+/*
-+ * Object layout:
-+ *
-+ * object address
-+ * Bytes of the object to be managed.
-+ * If the freepointer may overlay the object then the free
-+ * pointer is the first word of the object.
-+ *
-+ * Poisoning uses 0x6b (POISON_FREE) and the last byte is
-+ * 0xa5 (POISON_END)
-+ *
-+ * object + s->object_size
-+ * Padding to reach word boundary. This is also used for Redzoning.
-+ * Padding is extended by another word if Redzoning is enabled and
-+ * object_size == inuse.
-+ *
-+ * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
-+ * 0xcc (RED_ACTIVE) for objects in use.
-+ *
-+ * object + s->inuse
-+ * Meta data starts here.
-+ *
-+ * A. Free pointer (if we cannot overwrite object on free)
-+ * B. Tracking data for SLAB_STORE_USER
-+ * C. Padding to reach required alignment boundary or at mininum
-+ * one word if debugging is on to be able to detect writes
-+ * before the word boundary.
-+ *
-+ * Padding is done using 0x5a (POISON_INUSE)
-+ *
-+ * object + s->size
-+ * Nothing is used beyond s->size.
-+ *
-+ * If slabcaches are merged then the object_size and inuse boundaries are mostly
-+ * ignored. And therefore no slab options that rely on these boundaries
-+ * may be used with merged slabcaches.
-+ */
-+
-+static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
-+{
-+ unsigned long off = s->inuse; /* The end of info */
-+
-+ if (s->offset)
-+ /* Freepointer is placed after the object. */
-+ off += sizeof(void *);
-+
-+ if (s->flags & SLAB_STORE_USER)
-+ /* We also have user information there */
-+ off += 2 * sizeof(struct track);
-+
-+ if (s->size == off)
-+ return 1;
-+
-+ return check_bytes_and_report(s, page, p, "Object padding",
-+ p + off, POISON_INUSE, s->size - off);
-+}
-+
-+/* Check the pad bytes at the end of a slab page */
-+static int slab_pad_check(struct kmem_cache *s, struct page *page)
-+{
-+ u8 *start;
-+ u8 *fault;
-+ u8 *end;
-+ int length;
-+ int remainder;
-+
-+ if (!(s->flags & SLAB_POISON))
-+ return 1;
-+
-+ start = page_address(page);
-+ length = (PAGE_SIZE << compound_order(page)) - s->reserved;
-+ end = start + length;
-+ remainder = length % s->size;
-+ if (!remainder)
-+ return 1;
-+
-+ metadata_access_enable();
-+ fault = memchr_inv(end - remainder, POISON_INUSE, remainder);
-+ metadata_access_disable();
-+ if (!fault)
-+ return 1;
-+ while (end > fault && end[-1] == POISON_INUSE)
-+ end--;
-+
-+ slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
-+ print_section("Padding ", end - remainder, remainder);
-+
-+ restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
-+ return 0;
-+}
-+
-+static int check_object(struct kmem_cache *s, struct page *page,
-+ void *object, u8 val)
-+{
-+ u8 *p = object;
-+ u8 *endobject = object + s->object_size;
-+
-+ if (s->flags & SLAB_RED_ZONE) {
-+ if (!check_bytes_and_report(s, page, object, "Redzone",
-+ endobject, val, s->inuse - s->object_size))
-+ return 0;
-+ } else {
-+ if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
-+ check_bytes_and_report(s, page, p, "Alignment padding",
-+ endobject, POISON_INUSE,
-+ s->inuse - s->object_size);
-+ }
-+ }
-+
-+ if (s->flags & SLAB_POISON) {
-+ if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
-+ (!check_bytes_and_report(s, page, p, "Poison", p,
-+ POISON_FREE, s->object_size - 1) ||
-+ !check_bytes_and_report(s, page, p, "Poison",
-+ p + s->object_size - 1, POISON_END, 1)))
-+ return 0;
-+ /*
-+ * check_pad_bytes cleans up on its own.
-+ */
-+ check_pad_bytes(s, page, p);
-+ }
-+
-+ if (!s->offset && val == SLUB_RED_ACTIVE)
-+ /*
-+ * Object and freepointer overlap. Cannot check
-+ * freepointer while object is allocated.
-+ */
-+ return 1;
-+
-+ /* Check free pointer validity */
-+ if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
-+ object_err(s, page, p, "Freepointer corrupt");
-+ /*
-+ * No choice but to zap it and thus lose the remainder
-+ * of the free objects in this slab. May cause
-+ * another error because the object count is now wrong.
-+ */
-+ set_freepointer(s, p, NULL);
-+ return 0;
-+ }
-+ return 1;
-+}
-+
-+static int check_slab(struct kmem_cache *s, struct page *page)
-+{
-+ int maxobj;
-+
-+ VM_BUG_ON(!irqs_disabled());
-+
-+ if (!PageSlab(page)) {
-+ slab_err(s, page, "Not a valid slab page");
-+ return 0;
-+ }
-+
-+ maxobj = order_objects(compound_order(page), s->size, s->reserved);
-+ if (page->objects > maxobj) {
-+ slab_err(s, page, "objects %u > max %u",
-+ page->objects, maxobj);
-+ return 0;
-+ }
-+ if (page->inuse > page->objects) {
-+ slab_err(s, page, "inuse %u > max %u",
-+ page->inuse, page->objects);
-+ return 0;
-+ }
-+ /* Slab_pad_check fixes things up after itself */
-+ slab_pad_check(s, page);
-+ return 1;
-+}
-+
-+/*
-+ * Determine if a certain object on a page is on the freelist. Must hold the
-+ * slab lock to guarantee that the chains are in a consistent state.
-+ */
-+static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
-+{
-+ int nr = 0;
-+ void *fp;
-+ void *object = NULL;
-+ int max_objects;
-+
-+ fp = page->freelist;
-+ while (fp && nr <= page->objects) {
-+ if (fp == search)
-+ return 1;
-+ if (!check_valid_pointer(s, page, fp)) {
-+ if (object) {
-+ object_err(s, page, object,
-+ "Freechain corrupt");
-+ set_freepointer(s, object, NULL);
-+ } else {
-+ slab_err(s, page, "Freepointer corrupt");
-+ page->freelist = NULL;
-+ page->inuse = page->objects;
-+ slab_fix(s, "Freelist cleared");
-+ return 0;
-+ }
-+ break;
-+ }
-+ object = fp;
-+ fp = get_freepointer(s, object);
-+ nr++;
-+ }
-+
-+ max_objects = order_objects(compound_order(page), s->size, s->reserved);
-+ if (max_objects > MAX_OBJS_PER_PAGE)
-+ max_objects = MAX_OBJS_PER_PAGE;
-+
-+ if (page->objects != max_objects) {
-+ slab_err(s, page, "Wrong number of objects. Found %d but "
-+ "should be %d", page->objects, max_objects);
-+ page->objects = max_objects;
-+ slab_fix(s, "Number of objects adjusted.");
-+ }
-+ if (page->inuse != page->objects - nr) {
-+ slab_err(s, page, "Wrong object count. Counter is %d but "
-+ "counted were %d", page->inuse, page->objects - nr);
-+ page->inuse = page->objects - nr;
-+ slab_fix(s, "Object count adjusted.");
-+ }
-+ return search == NULL;
-+}
-+
-+static void trace(struct kmem_cache *s, struct page *page, void *object,
-+ int alloc)
-+{
-+ if (s->flags & SLAB_TRACE) {
-+ pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
-+ s->name,
-+ alloc ? "alloc" : "free",
-+ object, page->inuse,
-+ page->freelist);
-+
-+ if (!alloc)
-+ print_section("Object ", (void *)object,
-+ s->object_size);
-+
-+ dump_stack();
-+ }
-+}
-+
-+/*
-+ * Tracking of fully allocated slabs for debugging purposes.
-+ */
-+static void add_full(struct kmem_cache *s,
-+ struct kmem_cache_node *n, struct page *page)
-+{
-+ if (!(s->flags & SLAB_STORE_USER))
-+ return;
-+
-+ lockdep_assert_held(&n->list_lock);
-+ list_add(&page->lru, &n->full);
-+}
-+
-+static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
-+{
-+ if (!(s->flags & SLAB_STORE_USER))
-+ return;
-+
-+ lockdep_assert_held(&n->list_lock);
-+ list_del(&page->lru);
-+}
-+
-+/* Tracking of the number of slabs for debugging purposes */
-+static inline unsigned long slabs_node(struct kmem_cache *s, int node)
-+{
-+ struct kmem_cache_node *n = get_node(s, node);
-+
-+ return atomic_long_read(&n->nr_slabs);
-+}
-+
-+static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
-+{
-+ return atomic_long_read(&n->nr_slabs);
-+}
-+
-+static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
-+{
-+ struct kmem_cache_node *n = get_node(s, node);
-+
-+ /*
-+ * May be called early in order to allocate a slab for the
-+ * kmem_cache_node structure. Solve the chicken-egg
-+ * dilemma by deferring the increment of the count during
-+ * bootstrap (see early_kmem_cache_node_alloc).
-+ */
-+ if (likely(n)) {
-+ atomic_long_inc(&n->nr_slabs);
-+ atomic_long_add(objects, &n->total_objects);
-+ }
-+}
-+static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
-+{
-+ struct kmem_cache_node *n = get_node(s, node);
-+
-+ atomic_long_dec(&n->nr_slabs);
-+ atomic_long_sub(objects, &n->total_objects);
-+}
-+
-+/* Object debug checks for alloc/free paths */
-+static void setup_object_debug(struct kmem_cache *s, struct page *page,
-+ void *object)
-+{
-+ if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
-+ return;
-+
-+ init_object(s, object, SLUB_RED_INACTIVE);
-+ init_tracking(s, object);
-+}
-+
-+static noinline int alloc_debug_processing(struct kmem_cache *s,
-+ struct page *page,
-+ void *object, unsigned long addr)
-+{
-+ if (!check_slab(s, page))
-+ goto bad;
-+
-+ if (!check_valid_pointer(s, page, object)) {
-+ object_err(s, page, object, "Freelist Pointer check fails");
-+ goto bad;
-+ }
-+
-+ if (!check_object(s, page, object, SLUB_RED_INACTIVE))
-+ goto bad;
-+
-+ /* Success perform special debug activities for allocs */
-+ if (s->flags & SLAB_STORE_USER)
-+ set_track(s, object, TRACK_ALLOC, addr);
-+ trace(s, page, object, 1);
-+ init_object(s, object, SLUB_RED_ACTIVE);
-+ return 1;
-+
-+bad:
-+ if (PageSlab(page)) {
-+ /*
-+ * If this is a slab page then lets do the best we can
-+ * to avoid issues in the future. Marking all objects
-+ * as used avoids touching the remaining objects.
-+ */
-+ slab_fix(s, "Marking all objects used");
-+ page->inuse = page->objects;
-+ page->freelist = NULL;
-+ }
-+ return 0;
-+}
-+
-+static noinline struct kmem_cache_node *free_debug_processing(
-+ struct kmem_cache *s, struct page *page, void *object,
-+ unsigned long addr, unsigned long *flags)
-+{
-+ struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-+
-+ spin_lock_irqsave(&n->list_lock, *flags);
-+ slab_lock(page);
-+
-+ if (!check_slab(s, page))
-+ goto fail;
-+
-+ if (!check_valid_pointer(s, page, object)) {
-+ slab_err(s, page, "Invalid object pointer 0x%p", object);
-+ goto fail;
-+ }
-+
-+ if (on_freelist(s, page, object)) {
-+ object_err(s, page, object, "Object already free");
-+ goto fail;
-+ }
-+
-+ if (!check_object(s, page, object, SLUB_RED_ACTIVE))
-+ goto out;
-+
-+ if (unlikely(s != page->slab_cache)) {
-+ if (!PageSlab(page)) {
-+ slab_err(s, page, "Attempt to free object(0x%p) "
-+ "outside of slab", object);
-+ } else if (!page->slab_cache) {
-+ pr_err("SLUB <none>: no slab for object 0x%p.\n",
-+ object);
-+ dump_stack();
-+ } else
-+ object_err(s, page, object,
-+ "page slab pointer corrupt.");
-+ goto fail;
-+ }
-+
-+ if (s->flags & SLAB_STORE_USER)
-+ set_track(s, object, TRACK_FREE, addr);
-+ trace(s, page, object, 0);
-+ init_object(s, object, SLUB_RED_INACTIVE);
-+out:
-+ slab_unlock(page);
-+ /*
-+ * Keep node_lock to preserve integrity
-+ * until the object is actually freed
-+ */
-+ return n;
-+
-+fail:
-+ slab_unlock(page);
-+ spin_unlock_irqrestore(&n->list_lock, *flags);
-+ slab_fix(s, "Object at 0x%p not freed", object);
-+ return NULL;
-+}
-+
-+static int __init setup_slub_debug(char *str)
-+{
-+ slub_debug = DEBUG_DEFAULT_FLAGS;
-+ if (*str++ != '=' || !*str)
-+ /*
-+ * No options specified. Switch on full debugging.
-+ */
-+ goto out;
-+
-+ if (*str == ',')
-+ /*
-+ * No options but restriction on slabs. This means full
-+ * debugging for slabs matching a pattern.
-+ */
-+ goto check_slabs;
-+
-+ slub_debug = 0;
-+ if (*str == '-')
-+ /*
-+ * Switch off all debugging measures.
-+ */
-+ goto out;
-+
-+ /*
-+ * Determine which debug features should be switched on
-+ */
-+ for (; *str && *str != ','; str++) {
-+ switch (tolower(*str)) {
-+ case 'f':
-+ slub_debug |= SLAB_DEBUG_FREE;
-+ break;
-+ case 'z':
-+ slub_debug |= SLAB_RED_ZONE;
-+ break;
-+ case 'p':
-+ slub_debug |= SLAB_POISON;
-+ break;
-+ case 'u':
-+ slub_debug |= SLAB_STORE_USER;
-+ break;
-+ case 't':
-+ slub_debug |= SLAB_TRACE;
-+ break;
-+ case 'a':
-+ slub_debug |= SLAB_FAILSLAB;
-+ break;
-+ case 'o':
-+ /*
-+ * Avoid enabling debugging on caches if its minimum
-+ * order would increase as a result.
-+ */
-+ disable_higher_order_debug = 1;
-+ break;
-+ default:
-+ pr_err("slub_debug option '%c' unknown. skipped\n",
-+ *str);
-+ }
-+ }
-+
-+check_slabs:
-+ if (*str == ',')
-+ slub_debug_slabs = str + 1;
-+out:
-+ return 1;
-+}
-+
-+__setup("slub_debug", setup_slub_debug);
-+
-+unsigned long kmem_cache_flags(unsigned long object_size,
-+ unsigned long flags, const char *name,
-+ void (*ctor)(void *))
-+{
-+ /*
-+ * Enable debugging if selected on the kernel commandline.
-+ */
-+ if (slub_debug && (!slub_debug_slabs || (name &&
-+ !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)))))
-+ flags |= slub_debug;
-+
-+ return flags;
-+}
-+#else
-+static inline void setup_object_debug(struct kmem_cache *s,
-+ struct page *page, void *object) {}
-+
-+static inline int alloc_debug_processing(struct kmem_cache *s,
-+ struct page *page, void *object, unsigned long addr) { return 0; }
-+
-+static inline struct kmem_cache_node *free_debug_processing(
-+ struct kmem_cache *s, struct page *page, void *object,
-+ unsigned long addr, unsigned long *flags) { return NULL; }
-+
-+static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
-+ { return 1; }
-+static inline int check_object(struct kmem_cache *s, struct page *page,
-+ void *object, u8 val) { return 1; }
-+static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
-+ struct page *page) {}
-+static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
-+ struct page *page) {}
-+unsigned long kmem_cache_flags(unsigned long object_size,
-+ unsigned long flags, const char *name,
-+ void (*ctor)(void *))
-+{
-+ return flags;
-+}
-+#define slub_debug 0
-+
-+#define disable_higher_order_debug 0
-+
-+static inline unsigned long slabs_node(struct kmem_cache *s, int node)
-+ { return 0; }
-+static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
-+ { return 0; }
-+static inline void inc_slabs_node(struct kmem_cache *s, int node,
-+ int objects) {}
-+static inline void dec_slabs_node(struct kmem_cache *s, int node,
-+ int objects) {}
-+
-+#endif /* CONFIG_SLUB_DEBUG */
-+
-+/*
-+ * Hooks for other subsystems that check memory allocations. In a typical
-+ * production configuration these hooks all should produce no code at all.
-+ */
-+static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
-+{
-+ kmemleak_alloc(ptr, size, 1, flags);
-+ kasan_kmalloc_large(ptr, size);
-+}
-+
-+static inline void kfree_hook(const void *x)
-+{
-+ kmemleak_free(x);
-+ kasan_kfree_large(x);
-+}
-+
-+static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
-+ gfp_t flags)
-+{
-+ flags &= gfp_allowed_mask;
-+ lockdep_trace_alloc(flags);
-+ might_sleep_if(flags & __GFP_WAIT);
-+
-+ if (should_failslab(s->object_size, flags, s->flags))
-+ return NULL;
-+
-+ return memcg_kmem_get_cache(s, flags);
-+}
-+
-+static inline void slab_post_alloc_hook(struct kmem_cache *s,
-+ gfp_t flags, void *object)
-+{
-+ flags &= gfp_allowed_mask;
-+ kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
-+ kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags);
-+ memcg_kmem_put_cache(s);
-+ kasan_slab_alloc(s, object);
-+}
-+
-+static inline void slab_free_hook(struct kmem_cache *s, void *x)
-+{
-+ kmemleak_free_recursive(x, s->flags);
-+
-+ /*
-+ * Trouble is that we may no longer disable interrupts in the fast path
-+ * So in order to make the debug calls that expect irqs to be
-+ * disabled we need to disable interrupts temporarily.
-+ */
-+#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP)
-+ {
-+ unsigned long flags;
-+
-+ local_irq_save(flags);
-+ kmemcheck_slab_free(s, x, s->object_size);
-+ debug_check_no_locks_freed(x, s->object_size);
-+ local_irq_restore(flags);
-+ }
-+#endif
-+ if (!(s->flags & SLAB_DEBUG_OBJECTS))
-+ debug_check_no_obj_freed(x, s->object_size);
-+
-+ kasan_slab_free(s, x);
-+}
-+
-+/*
-+ * Slab allocation and freeing
-+ */
-+static inline struct page *alloc_slab_page(struct kmem_cache *s,
-+ gfp_t flags, int node, struct kmem_cache_order_objects oo)
-+{
-+ struct page *page;
-+ int order = oo_order(oo);
-+
-+ flags |= __GFP_NOTRACK;
-+
-+ if (memcg_charge_slab(s, flags, order))
-+ return NULL;
-+
-+ if (node == NUMA_NO_NODE)
-+ page = alloc_pages(flags, order);
-+ else
-+ page = alloc_pages_exact_node(node, flags, order);
-+
-+ if (!page)
-+ memcg_uncharge_slab(s, order);
-+
-+ return page;
-+}
-+
-+static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
-+{
-+ struct page *page;
-+ struct kmem_cache_order_objects oo = s->oo;
-+ gfp_t alloc_gfp;
-+
-+ flags &= gfp_allowed_mask;
-+
-+ if (flags & __GFP_WAIT)
-+ local_irq_enable();
-+
-+ flags |= s->allocflags;
-+
-+ /*
-+ * Let the initial higher-order allocation fail under memory pressure
-+ * so we fall-back to the minimum order allocation.
-+ */
-+ alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
-+
-+ page = alloc_slab_page(s, alloc_gfp, node, oo);
-+ if (unlikely(!page)) {
-+ oo = s->min;
-+ alloc_gfp = flags;
-+ /*
-+ * Allocation may have failed due to fragmentation.
-+ * Try a lower order alloc if possible
-+ */
-+ page = alloc_slab_page(s, alloc_gfp, node, oo);
-+
-+ if (page)
-+ stat(s, ORDER_FALLBACK);
-+ }
-+
-+ if (kmemcheck_enabled && page
-+ && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
-+ int pages = 1 << oo_order(oo);
-+
-+ kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node);
-+
-+ /*
-+ * Objects from caches that have a constructor don't get
-+ * cleared when they're allocated, so we need to do it here.
-+ */
-+ if (s->ctor)
-+ kmemcheck_mark_uninitialized_pages(page, pages);
-+ else
-+ kmemcheck_mark_unallocated_pages(page, pages);
-+ }
-+
-+ if (flags & __GFP_WAIT)
-+ local_irq_disable();
-+ if (!page)
-+ return NULL;
-+
-+ page->objects = oo_objects(oo);
-+ mod_zone_page_state(page_zone(page),
-+ (s->flags & SLAB_RECLAIM_ACCOUNT) ?
-+ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
-+ 1 << oo_order(oo));
-+
-+ return page;
-+}
-+
-+static void setup_object(struct kmem_cache *s, struct page *page,
-+ void *object)
-+{
-+ setup_object_debug(s, page, object);
-+ if (unlikely(s->ctor)) {
-+ kasan_unpoison_object_data(s, object);
-+ s->ctor(object);
-+ kasan_poison_object_data(s, object);
-+ }
-+}
-+
-+static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
-+{
-+ struct page *page;
-+ void *start;
-+ void *p;
-+ int order;
-+ int idx;
-+
-+ if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
-+ pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
-+ BUG();
-+ }
-+
-+ page = allocate_slab(s,
-+ flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
-+ if (!page)
-+ goto out;
-+
-+ order = compound_order(page);
-+ inc_slabs_node(s, page_to_nid(page), page->objects);
-+ page->slab_cache = s;
-+ __SetPageSlab(page);
-+ if (page_is_pfmemalloc(page))
-+ SetPageSlabPfmemalloc(page);
-+
-+ start = page_address(page);
-+
-+ if (unlikely(s->flags & SLAB_POISON))
-+ memset(start, POISON_INUSE, PAGE_SIZE << order);
-+
-+ kasan_poison_slab(page);
-+
-+ for_each_object_idx(p, idx, s, start, page->objects) {
-+ setup_object(s, page, p);
-+ if (likely(idx < page->objects))
-+ set_freepointer(s, p, p + s->size);
-+ else
-+ set_freepointer(s, p, NULL);
-+ }
-+
-+ page->freelist = start;
-+ page->inuse = page->objects;
-+ page->frozen = 1;
-+out:
-+ return page;
-+}
-+
-+static void __free_slab(struct kmem_cache *s, struct page *page)
-+{
-+ int order = compound_order(page);
-+ int pages = 1 << order;
-+
-+ if (kmem_cache_debug(s)) {
-+ void *p;
-+
-+ slab_pad_check(s, page);
-+ for_each_object(p, s, page_address(page),
-+ page->objects)
-+ check_object(s, page, p, SLUB_RED_INACTIVE);
-+ }
-+
-+ kmemcheck_free_shadow(page, compound_order(page));
-+
-+ mod_zone_page_state(page_zone(page),
-+ (s->flags & SLAB_RECLAIM_ACCOUNT) ?
-+ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
-+ -pages);
-+
-+ __ClearPageSlabPfmemalloc(page);
-+ __ClearPageSlab(page);
-+
-+ page_mapcount_reset(page);
-+ if (current->reclaim_state)
-+ current->reclaim_state->reclaimed_slab += pages;
-+ __free_pages(page, order);
-+ memcg_uncharge_slab(s, order);
-+}
-+
-+#define need_reserve_slab_rcu \
-+ (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
-+
-+static void rcu_free_slab(struct rcu_head *h)
-+{
-+ struct page *page;
-+
-+ if (need_reserve_slab_rcu)
-+ page = virt_to_head_page(h);
-+ else
-+ page = container_of((struct list_head *)h, struct page, lru);
-+
-+ __free_slab(page->slab_cache, page);
-+}
-+
-+static void free_slab(struct kmem_cache *s, struct page *page)
-+{
-+ if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
-+ struct rcu_head *head;
-+
-+ if (need_reserve_slab_rcu) {
-+ int order = compound_order(page);
-+ int offset = (PAGE_SIZE << order) - s->reserved;
-+
-+ VM_BUG_ON(s->reserved != sizeof(*head));
-+ head = page_address(page) + offset;
-+ } else {
-+ /*
-+ * RCU free overloads the RCU head over the LRU
-+ */
-+ head = (void *)&page->lru;
-+ }
-+
-+ call_rcu(head, rcu_free_slab);
-+ } else
-+ __free_slab(s, page);
-+}
-+
-+static void discard_slab(struct kmem_cache *s, struct page *page)
-+{
-+ dec_slabs_node(s, page_to_nid(page), page->objects);
-+ free_slab(s, page);
-+}
-+
-+/*
-+ * Management of partially allocated slabs.
-+ */
-+static inline void
-+__add_partial(struct kmem_cache_node *n, struct page *page, int tail)
-+{
-+ n->nr_partial++;
-+ if (tail == DEACTIVATE_TO_TAIL)
-+ list_add_tail(&page->lru, &n->partial);
-+ else
-+ list_add(&page->lru, &n->partial);
-+}
-+
-+static inline void add_partial(struct kmem_cache_node *n,
-+ struct page *page, int tail)
-+{
-+ lockdep_assert_held(&n->list_lock);
-+ __add_partial(n, page, tail);
-+}
-+
-+static inline void
-+__remove_partial(struct kmem_cache_node *n, struct page *page)
-+{
-+ list_del(&page->lru);
-+ n->nr_partial--;
-+}
-+
-+static inline void remove_partial(struct kmem_cache_node *n,
-+ struct page *page)
-+{
-+ lockdep_assert_held(&n->list_lock);
-+ __remove_partial(n, page);
-+}
-+
-+/*
-+ * Remove slab from the partial list, freeze it and
-+ * return the pointer to the freelist.
-+ *
-+ * Returns a list of objects or NULL if it fails.
-+ */
-+static inline void *acquire_slab(struct kmem_cache *s,
-+ struct kmem_cache_node *n, struct page *page,
-+ int mode, int *objects)
-+{
-+ void *freelist;
-+ unsigned long counters;
-+ struct page new;
-+
-+ lockdep_assert_held(&n->list_lock);
-+
-+ /*
-+ * Zap the freelist and set the frozen bit.
-+ * The old freelist is the list of objects for the
-+ * per cpu allocation list.
-+ */
-+ freelist = page->freelist;
-+ counters = page->counters;
-+ new.counters = counters;
-+ *objects = new.objects - new.inuse;
-+ if (mode) {
-+ new.inuse = page->objects;
-+ new.freelist = NULL;
-+ } else {
-+ new.freelist = freelist;
-+ }
-+
-+ VM_BUG_ON(new.frozen);
-+ new.frozen = 1;
-+
-+ if (!__cmpxchg_double_slab(s, page,
-+ freelist, counters,
-+ new.freelist, new.counters,
-+ "acquire_slab"))
-+ return NULL;
-+
-+ remove_partial(n, page);
-+ WARN_ON(!freelist);
-+ return freelist;
-+}
-+
-+static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
-+static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
-+
-+/*
-+ * Try to allocate a partial slab from a specific node.
-+ */
-+static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
-+ struct kmem_cache_cpu *c, gfp_t flags)
-+{
-+ struct page *page, *page2;
-+ void *object = NULL;
-+ int available = 0;
-+ int objects;
-+
-+ /*
-+ * Racy check. If we mistakenly see no partial slabs then we
-+ * just allocate an empty slab. If we mistakenly try to get a
-+ * partial slab and there is none available then get_partials()
-+ * will return NULL.
-+ */
-+ if (!n || !n->nr_partial)
-+ return NULL;
-+
-+ spin_lock(&n->list_lock);
-+ list_for_each_entry_safe(page, page2, &n->partial, lru) {
-+ void *t;
-+
-+ if (!pfmemalloc_match(page, flags))
-+ continue;
-+
-+ t = acquire_slab(s, n, page, object == NULL, &objects);
-+ if (!t)
-+ break;
-+
-+ available += objects;
-+ if (!object) {
-+ c->page = page;
-+ stat(s, ALLOC_FROM_PARTIAL);
-+ object = t;
-+ } else {
-+ put_cpu_partial(s, page, 0);
-+ stat(s, CPU_PARTIAL_NODE);
-+ }
-+ if (!kmem_cache_has_cpu_partial(s)
-+ || available > s->cpu_partial / 2)
-+ break;
-+
-+ }
-+ spin_unlock(&n->list_lock);
-+ return object;
-+}
-+
-+/*
-+ * Get a page from somewhere. Search in increasing NUMA distances.
-+ */
-+static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
-+ struct kmem_cache_cpu *c)
-+{
-+#ifdef CONFIG_NUMA
-+ struct zonelist *zonelist;
-+ struct zoneref *z;
-+ struct zone *zone;
-+ enum zone_type high_zoneidx = gfp_zone(flags);
-+ void *object;
-+ unsigned int cpuset_mems_cookie;
-+
-+ /*
-+ * The defrag ratio allows a configuration of the tradeoffs between
-+ * inter node defragmentation and node local allocations. A lower
-+ * defrag_ratio increases the tendency to do local allocations
-+ * instead of attempting to obtain partial slabs from other nodes.
-+ *
-+ * If the defrag_ratio is set to 0 then kmalloc() always
-+ * returns node local objects. If the ratio is higher then kmalloc()
-+ * may return off node objects because partial slabs are obtained
-+ * from other nodes and filled up.
-+ *
-+ * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
-+ * defrag_ratio = 1000) then every (well almost) allocation will
-+ * first attempt to defrag slab caches on other nodes. This means
-+ * scanning over all nodes to look for partial slabs which may be
-+ * expensive if we do it every time we are trying to find a slab
-+ * with available objects.
-+ */
-+ if (!s->remote_node_defrag_ratio ||
-+ get_cycles() % 1024 > s->remote_node_defrag_ratio)
-+ return NULL;
-+
-+ do {
-+ cpuset_mems_cookie = read_mems_allowed_begin();
-+ zonelist = node_zonelist(mempolicy_slab_node(), flags);
-+ for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
-+ struct kmem_cache_node *n;
-+
-+ n = get_node(s, zone_to_nid(zone));
-+
-+ if (n && cpuset_zone_allowed(zone, flags) &&
-+ n->nr_partial > s->min_partial) {
-+ object = get_partial_node(s, n, c, flags);
-+ if (object) {
-+ /*
-+ * Don't check read_mems_allowed_retry()
-+ * here - if mems_allowed was updated in
-+ * parallel, that was a harmless race
-+ * between allocation and the cpuset
-+ * update
-+ */
-+ return object;
-+ }
-+ }
-+ }
-+ } while (read_mems_allowed_retry(cpuset_mems_cookie));
-+#endif
-+ return NULL;
-+}
-+
-+/*
-+ * Get a partial page, lock it and return it.
-+ */
-+static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
-+ struct kmem_cache_cpu *c)
-+{
-+ void *object;
-+ int searchnode = node;
-+
-+ if (node == NUMA_NO_NODE)
-+ searchnode = numa_mem_id();
-+ else if (!node_present_pages(node))
-+ searchnode = node_to_mem_node(node);
-+
-+ object = get_partial_node(s, get_node(s, searchnode), c, flags);
-+ if (object || node != NUMA_NO_NODE)
-+ return object;
-+
-+ return get_any_partial(s, flags, c);
-+}
-+
-+#ifdef CONFIG_PREEMPT
-+/*
-+ * Calculate the next globally unique transaction for disambiguiation
-+ * during cmpxchg. The transactions start with the cpu number and are then
-+ * incremented by CONFIG_NR_CPUS.
-+ */
-+#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
-+#else
-+/*
-+ * No preemption supported therefore also no need to check for
-+ * different cpus.
-+ */
-+#define TID_STEP 1
-+#endif
-+
-+static inline unsigned long next_tid(unsigned long tid)
-+{
-+ return tid + TID_STEP;
-+}
-+
-+static inline unsigned int tid_to_cpu(unsigned long tid)
-+{
-+ return tid % TID_STEP;
-+}
-+
-+static inline unsigned long tid_to_event(unsigned long tid)
-+{
-+ return tid / TID_STEP;
-+}
-+
-+static inline unsigned int init_tid(int cpu)
-+{
-+ return cpu;
-+}
-+
-+static inline void note_cmpxchg_failure(const char *n,
-+ const struct kmem_cache *s, unsigned long tid)
-+{
-+#ifdef SLUB_DEBUG_CMPXCHG
-+ unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
-+
-+ pr_info("%s %s: cmpxchg redo ", n, s->name);
-+
-+#ifdef CONFIG_PREEMPT
-+ if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
-+ pr_warn("due to cpu change %d -> %d\n",
-+ tid_to_cpu(tid), tid_to_cpu(actual_tid));
-+ else
-+#endif
-+ if (tid_to_event(tid) != tid_to_event(actual_tid))
-+ pr_warn("due to cpu running other code. Event %ld->%ld\n",
-+ tid_to_event(tid), tid_to_event(actual_tid));
-+ else
-+ pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
-+ actual_tid, tid, next_tid(tid));
-+#endif
-+ stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
-+}
-+
-+static void init_kmem_cache_cpus(struct kmem_cache *s)
-+{
-+ int cpu;
-+
-+ for_each_possible_cpu(cpu)
-+ per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
-+}
-+
-+/*
-+ * Remove the cpu slab
-+ */
-+static void deactivate_slab(struct kmem_cache *s, struct page *page,
-+ void *freelist)
-+{
-+ enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
-+ struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-+ int lock = 0;
-+ enum slab_modes l = M_NONE, m = M_NONE;
-+ void *nextfree;
-+ int tail = DEACTIVATE_TO_HEAD;
-+ struct page new;
-+ struct page old;
-+
-+ if (page->freelist) {
-+ stat(s, DEACTIVATE_REMOTE_FREES);
-+ tail = DEACTIVATE_TO_TAIL;
-+ }
-+
-+ /*
-+ * Stage one: Free all available per cpu objects back
-+ * to the page freelist while it is still frozen. Leave the
-+ * last one.
-+ *
-+ * There is no need to take the list->lock because the page
-+ * is still frozen.
-+ */
-+ while (freelist && (nextfree = get_freepointer(s, freelist))) {
-+ void *prior;
-+ unsigned long counters;
-+
-+ do {
-+ prior = page->freelist;
-+ counters = page->counters;
-+ set_freepointer(s, freelist, prior);
-+ new.counters = counters;
-+ new.inuse--;
-+ VM_BUG_ON(!new.frozen);
-+
-+ } while (!__cmpxchg_double_slab(s, page,
-+ prior, counters,
-+ freelist, new.counters,
-+ "drain percpu freelist"));
-+
-+ freelist = nextfree;
-+ }
-+
-+ /*
-+ * Stage two: Ensure that the page is unfrozen while the
-+ * list presence reflects the actual number of objects
-+ * during unfreeze.
-+ *
-+ * We setup the list membership and then perform a cmpxchg
-+ * with the count. If there is a mismatch then the page
-+ * is not unfrozen but the page is on the wrong list.
-+ *
-+ * Then we restart the process which may have to remove
-+ * the page from the list that we just put it on again
-+ * because the number of objects in the slab may have
-+ * changed.
-+ */
-+redo:
-+
-+ old.freelist = page->freelist;
-+ old.counters = page->counters;
-+ VM_BUG_ON(!old.frozen);
-+
-+ /* Determine target state of the slab */
-+ new.counters = old.counters;
-+ if (freelist) {
-+ new.inuse--;
-+ set_freepointer(s, freelist, old.freelist);
-+ new.freelist = freelist;
-+ } else
-+ new.freelist = old.freelist;
-+
-+ new.frozen = 0;
-+
-+ if (!new.inuse && n->nr_partial >= s->min_partial)
-+ m = M_FREE;
-+ else if (new.freelist) {
-+ m = M_PARTIAL;
-+ if (!lock) {
-+ lock = 1;
-+ /*
-+ * Taking the spinlock removes the possiblity
-+ * that acquire_slab() will see a slab page that
-+ * is frozen
-+ */
-+ spin_lock(&n->list_lock);
-+ }
-+ } else {
-+ m = M_FULL;
-+ if (kmem_cache_debug(s) && !lock) {
-+ lock = 1;
-+ /*
-+ * This also ensures that the scanning of full
-+ * slabs from diagnostic functions will not see
-+ * any frozen slabs.
-+ */
-+ spin_lock(&n->list_lock);
-+ }
-+ }
-+
-+ if (l != m) {
-+
-+ if (l == M_PARTIAL)
-+
-+ remove_partial(n, page);
-+
-+ else if (l == M_FULL)
-+
-+ remove_full(s, n, page);
-+
-+ if (m == M_PARTIAL) {
-+
-+ add_partial(n, page, tail);
-+ stat(s, tail);
-+
-+ } else if (m == M_FULL) {
-+
-+ stat(s, DEACTIVATE_FULL);
-+ add_full(s, n, page);
-+
-+ }
-+ }
-+
-+ l = m;
-+ if (!__cmpxchg_double_slab(s, page,
-+ old.freelist, old.counters,
-+ new.freelist, new.counters,
-+ "unfreezing slab"))
-+ goto redo;
-+
-+ if (lock)
-+ spin_unlock(&n->list_lock);
-+
-+ if (m == M_FREE) {
-+ stat(s, DEACTIVATE_EMPTY);
-+ discard_slab(s, page);
-+ stat(s, FREE_SLAB);
-+ }
-+}
-+
-+/*
-+ * Unfreeze all the cpu partial slabs.
-+ *
-+ * This function must be called with interrupts disabled
-+ * for the cpu using c (or some other guarantee must be there
-+ * to guarantee no concurrent accesses).
-+ */
-+static void unfreeze_partials(struct kmem_cache *s,
-+ struct kmem_cache_cpu *c)
-+{
-+#ifdef CONFIG_SLUB_CPU_PARTIAL
-+ struct kmem_cache_node *n = NULL, *n2 = NULL;
-+ struct page *page, *discard_page = NULL;
-+
-+ while ((page = c->partial)) {
-+ struct page new;
-+ struct page old;
-+
-+ c->partial = page->next;
-+
-+ n2 = get_node(s, page_to_nid(page));
-+ if (n != n2) {
-+ if (n)
-+ spin_unlock(&n->list_lock);
-+
-+ n = n2;
-+ spin_lock(&n->list_lock);
-+ }
-+
-+ do {
-+
-+ old.freelist = page->freelist;
-+ old.counters = page->counters;
-+ VM_BUG_ON(!old.frozen);
-+
-+ new.counters = old.counters;
-+ new.freelist = old.freelist;
-+
-+ new.frozen = 0;
-+
-+ } while (!__cmpxchg_double_slab(s, page,
-+ old.freelist, old.counters,
-+ new.freelist, new.counters,
-+ "unfreezing slab"));
-+
-+ if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
-+ page->next = discard_page;
-+ discard_page = page;
-+ } else {
-+ add_partial(n, page, DEACTIVATE_TO_TAIL);
-+ stat(s, FREE_ADD_PARTIAL);
-+ }
-+ }
-+
-+ if (n)
-+ spin_unlock(&n->list_lock);
-+
-+ while (discard_page) {
-+ page = discard_page;
-+ discard_page = discard_page->next;
-+
-+ stat(s, DEACTIVATE_EMPTY);
-+ discard_slab(s, page);
-+ stat(s, FREE_SLAB);
-+ }
-+#endif
-+}
-+
-+/*
-+ * Put a page that was just frozen (in __slab_free) into a partial page
-+ * slot if available. This is done without interrupts disabled and without
-+ * preemption disabled. The cmpxchg is racy and may put the partial page
-+ * onto a random cpus partial slot.
-+ *
-+ * If we did not find a slot then simply move all the partials to the
-+ * per node partial list.
-+ */
-+static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
-+{
-+#ifdef CONFIG_SLUB_CPU_PARTIAL
-+ struct page *oldpage;
-+ int pages;
-+ int pobjects;
-+
-+ preempt_disable();
-+ do {
-+ pages = 0;
-+ pobjects = 0;
-+ oldpage = this_cpu_read(s->cpu_slab->partial);
-+
-+ if (oldpage) {
-+ pobjects = oldpage->pobjects;
-+ pages = oldpage->pages;
-+ if (drain && pobjects > s->cpu_partial) {
-+ unsigned long flags;
-+ /*
-+ * partial array is full. Move the existing
-+ * set to the per node partial list.
-+ */
-+ local_irq_save(flags);
-+ unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
-+ local_irq_restore(flags);
-+ oldpage = NULL;
-+ pobjects = 0;
-+ pages = 0;
-+ stat(s, CPU_PARTIAL_DRAIN);
-+ }
-+ }
-+
-+ pages++;
-+ pobjects += page->objects - page->inuse;
-+
-+ page->pages = pages;
-+ page->pobjects = pobjects;
-+ page->next = oldpage;
-+
-+ } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
-+ != oldpage);
-+ if (unlikely(!s->cpu_partial)) {
-+ unsigned long flags;
-+
-+ local_irq_save(flags);
-+ unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
-+ local_irq_restore(flags);
-+ }
-+ preempt_enable();
-+#endif
-+}
-+
-+static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
-+{
-+ stat(s, CPUSLAB_FLUSH);
-+ deactivate_slab(s, c->page, c->freelist);
-+
-+ c->tid = next_tid(c->tid);
-+ c->page = NULL;
-+ c->freelist = NULL;
-+}
-+
-+/*
-+ * Flush cpu slab.
-+ *
-+ * Called from IPI handler with interrupts disabled.
-+ */
-+static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
-+{
-+ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
-+
-+ if (likely(c)) {
-+ if (c->page)
-+ flush_slab(s, c);
-+
-+ unfreeze_partials(s, c);
-+ }
-+}
-+
-+static void flush_cpu_slab(void *d)
-+{
-+ struct kmem_cache *s = d;
-+
-+ __flush_cpu_slab(s, smp_processor_id());
-+}
-+
-+static bool has_cpu_slab(int cpu, void *info)
-+{
-+ struct kmem_cache *s = info;
-+ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
-+
-+ return c->page || c->partial;
-+}
-+
-+static void flush_all(struct kmem_cache *s)
-+{
-+ on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
-+}
-+
-+/*
-+ * Check if the objects in a per cpu structure fit numa
-+ * locality expectations.
-+ */
-+static inline int node_match(struct page *page, int node)
-+{
-+#ifdef CONFIG_NUMA
-+ if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node))
-+ return 0;
-+#endif
-+ return 1;
-+}
-+
-+#ifdef CONFIG_SLUB_DEBUG
-+static int count_free(struct page *page)
-+{
-+ return page->objects - page->inuse;
-+}
-+
-+static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
-+{
-+ return atomic_long_read(&n->total_objects);
-+}
-+#endif /* CONFIG_SLUB_DEBUG */
-+
-+#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
-+static unsigned long count_partial(struct kmem_cache_node *n,
-+ int (*get_count)(struct page *))
-+{
-+ unsigned long flags;
-+ unsigned long x = 0;
-+ struct page *page;
-+
-+ spin_lock_irqsave(&n->list_lock, flags);
-+ list_for_each_entry(page, &n->partial, lru)
-+ x += get_count(page);
-+ spin_unlock_irqrestore(&n->list_lock, flags);
-+ return x;
-+}
-+#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
-+
-+static noinline void
-+slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
-+{
-+#ifdef CONFIG_SLUB_DEBUG
-+ static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
-+ DEFAULT_RATELIMIT_BURST);
-+ int node;
-+ struct kmem_cache_node *n;
-+
-+ if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
-+ return;
-+
-+ pr_warn("SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
-+ nid, gfpflags);
-+ pr_warn(" cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n",
-+ s->name, s->object_size, s->size, oo_order(s->oo),
-+ oo_order(s->min));
-+
-+ if (oo_order(s->min) > get_order(s->object_size))
-+ pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n",
-+ s->name);
-+
-+ for_each_kmem_cache_node(s, node, n) {
-+ unsigned long nr_slabs;
-+ unsigned long nr_objs;
-+ unsigned long nr_free;
-+
-+ nr_free = count_partial(n, count_free);
-+ nr_slabs = node_nr_slabs(n);
-+ nr_objs = node_nr_objs(n);
-+
-+ pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
-+ node, nr_slabs, nr_objs, nr_free);
-+ }
-+#endif
-+}
-+
-+static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
-+ int node, struct kmem_cache_cpu **pc)
-+{
-+ void *freelist;
-+ struct kmem_cache_cpu *c = *pc;
-+ struct page *page;
-+
-+ freelist = get_partial(s, flags, node, c);
-+
-+ if (freelist)
-+ return freelist;
-+
-+ page = new_slab(s, flags, node);
-+ if (page) {
-+ c = raw_cpu_ptr(s->cpu_slab);
-+ if (c->page)
-+ flush_slab(s, c);
-+
-+ /*
-+ * No other reference to the page yet so we can
-+ * muck around with it freely without cmpxchg
-+ */
-+ freelist = page->freelist;
-+ page->freelist = NULL;
-+
-+ stat(s, ALLOC_SLAB);
-+ c->page = page;
-+ *pc = c;
-+ } else
-+ freelist = NULL;
-+
-+ return freelist;
-+}
-+
-+static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
-+{
-+ if (unlikely(PageSlabPfmemalloc(page)))
-+ return gfp_pfmemalloc_allowed(gfpflags);
-+
-+ return true;
-+}
-+
-+/*
-+ * Check the page->freelist of a page and either transfer the freelist to the
-+ * per cpu freelist or deactivate the page.
-+ *
-+ * The page is still frozen if the return value is not NULL.
-+ *
-+ * If this function returns NULL then the page has been unfrozen.
-+ *
-+ * This function must be called with interrupt disabled.
-+ */
-+static inline void *get_freelist(struct kmem_cache *s, struct page *page)
-+{
-+ struct page new;
-+ unsigned long counters;
-+ void *freelist;
-+
-+ do {
-+ freelist = page->freelist;
-+ counters = page->counters;
-+
-+ new.counters = counters;
-+ VM_BUG_ON(!new.frozen);
-+
-+ new.inuse = page->objects;
-+ new.frozen = freelist != NULL;
-+
-+ } while (!__cmpxchg_double_slab(s, page,
-+ freelist, counters,
-+ NULL, new.counters,
-+ "get_freelist"));
-+
-+ return freelist;
-+}
-+
-+/*
-+ * Slow path. The lockless freelist is empty or we need to perform
-+ * debugging duties.
-+ *
-+ * Processing is still very fast if new objects have been freed to the
-+ * regular freelist. In that case we simply take over the regular freelist
-+ * as the lockless freelist and zap the regular freelist.
-+ *
-+ * If that is not working then we fall back to the partial lists. We take the
-+ * first element of the freelist as the object to allocate now and move the
-+ * rest of the freelist to the lockless freelist.
-+ *
-+ * And if we were unable to get a new slab from the partial slab lists then
-+ * we need to allocate a new slab. This is the slowest path since it involves
-+ * a call to the page allocator and the setup of a new slab.
-+ */
-+static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
-+ unsigned long addr, struct kmem_cache_cpu *c)
-+{
-+ void *freelist;
-+ struct page *page;
-+ unsigned long flags;
-+
-+ local_irq_save(flags);
-+#ifdef CONFIG_PREEMPT
-+ /*
-+ * We may have been preempted and rescheduled on a different
-+ * cpu before disabling interrupts. Need to reload cpu area
-+ * pointer.
-+ */
-+ c = this_cpu_ptr(s->cpu_slab);
-+#endif
-+
-+ page = c->page;
-+ if (!page)
-+ goto new_slab;
-+redo:
-+
-+ if (unlikely(!node_match(page, node))) {
-+ int searchnode = node;
-+
-+ if (node != NUMA_NO_NODE && !node_present_pages(node))
-+ searchnode = node_to_mem_node(node);
-+
-+ if (unlikely(!node_match(page, searchnode))) {
-+ stat(s, ALLOC_NODE_MISMATCH);
-+ deactivate_slab(s, page, c->freelist);
-+ c->page = NULL;
-+ c->freelist = NULL;
-+ goto new_slab;
-+ }
-+ }
-+
-+ /*
-+ * By rights, we should be searching for a slab page that was
-+ * PFMEMALLOC but right now, we are losing the pfmemalloc
-+ * information when the page leaves the per-cpu allocator
-+ */
-+ if (unlikely(!pfmemalloc_match(page, gfpflags))) {
-+ deactivate_slab(s, page, c->freelist);
-+ c->page = NULL;
-+ c->freelist = NULL;
-+ goto new_slab;
-+ }
-+
-+ /* must check again c->freelist in case of cpu migration or IRQ */
-+ freelist = c->freelist;
-+ if (freelist)
-+ goto load_freelist;
-+
-+ freelist = get_freelist(s, page);
-+
-+ if (!freelist) {
-+ c->page = NULL;
-+ stat(s, DEACTIVATE_BYPASS);
-+ goto new_slab;
-+ }
-+
-+ stat(s, ALLOC_REFILL);
-+
-+load_freelist:
-+ /*
-+ * freelist is pointing to the list of objects to be used.
-+ * page is pointing to the page from which the objects are obtained.
-+ * That page must be frozen for per cpu allocations to work.
-+ */
-+ VM_BUG_ON(!c->page->frozen);
-+ c->freelist = get_freepointer(s, freelist);
-+ c->tid = next_tid(c->tid);
-+ local_irq_restore(flags);
-+ return freelist;
-+
-+new_slab:
-+
-+ if (c->partial) {
-+ page = c->page = c->partial;
-+ c->partial = page->next;
-+ stat(s, CPU_PARTIAL_ALLOC);
-+ c->freelist = NULL;
-+ goto redo;
-+ }
-+
-+ freelist = new_slab_objects(s, gfpflags, node, &c);
-+
-+ if (unlikely(!freelist)) {
-+ slab_out_of_memory(s, gfpflags, node);
-+ local_irq_restore(flags);
-+ return NULL;
-+ }
-+
-+ page = c->page;
-+ if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
-+ goto load_freelist;
-+
-+ /* Only entered in the debug case */
-+ if (kmem_cache_debug(s) &&
-+ !alloc_debug_processing(s, page, freelist, addr))
-+ goto new_slab; /* Slab failed checks. Next slab needed */
-+
-+ deactivate_slab(s, page, get_freepointer(s, freelist));
-+ c->page = NULL;
-+ c->freelist = NULL;
-+ local_irq_restore(flags);
-+ return freelist;
-+}
-+
-+/*
-+ * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
-+ * have the fastpath folded into their functions. So no function call
-+ * overhead for requests that can be satisfied on the fastpath.
-+ *
-+ * The fastpath works by first checking if the lockless freelist can be used.
-+ * If not then __slab_alloc is called for slow processing.
-+ *
-+ * Otherwise we can simply pick the next object from the lockless free list.
-+ */
-+static __always_inline void *slab_alloc_node(struct kmem_cache *s,
-+ gfp_t gfpflags, int node, unsigned long addr)
-+{
-+ void **object;
-+ struct kmem_cache_cpu *c;
-+ struct page *page;
-+ unsigned long tid;
-+
-+ s = slab_pre_alloc_hook(s, gfpflags);
-+ if (!s)
-+ return NULL;
-+redo:
-+ /*
-+ * Must read kmem_cache cpu data via this cpu ptr. Preemption is
-+ * enabled. We may switch back and forth between cpus while
-+ * reading from one cpu area. That does not matter as long
-+ * as we end up on the original cpu again when doing the cmpxchg.
-+ *
-+ * We should guarantee that tid and kmem_cache are retrieved on
-+ * the same cpu. It could be different if CONFIG_PREEMPT so we need
-+ * to check if it is matched or not.
-+ */
-+ do {
-+ tid = this_cpu_read(s->cpu_slab->tid);
-+ c = raw_cpu_ptr(s->cpu_slab);
-+ } while (IS_ENABLED(CONFIG_PREEMPT) &&
-+ unlikely(tid != READ_ONCE(c->tid)));
-+
-+ /*
-+ * Irqless object alloc/free algorithm used here depends on sequence
-+ * of fetching cpu_slab's data. tid should be fetched before anything
-+ * on c to guarantee that object and page associated with previous tid
-+ * won't be used with current tid. If we fetch tid first, object and
-+ * page could be one associated with next tid and our alloc/free
-+ * request will be failed. In this case, we will retry. So, no problem.
-+ */
-+ barrier();
-+
-+ /*
-+ * The transaction ids are globally unique per cpu and per operation on
-+ * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
-+ * occurs on the right processor and that there was no operation on the
-+ * linked list in between.
-+ */
-+
-+ object = c->freelist;
-+ page = c->page;
-+ if (unlikely(!object || !node_match(page, node))) {
-+ object = __slab_alloc(s, gfpflags, node, addr, c);
-+ stat(s, ALLOC_SLOWPATH);
-+ } else {
-+ void *next_object = get_freepointer_safe(s, object);
-+
-+ /*
-+ * The cmpxchg will only match if there was no additional
-+ * operation and if we are on the right processor.
-+ *
-+ * The cmpxchg does the following atomically (without lock
-+ * semantics!)
-+ * 1. Relocate first pointer to the current per cpu area.
-+ * 2. Verify that tid and freelist have not been changed
-+ * 3. If they were not changed replace tid and freelist
-+ *
-+ * Since this is without lock semantics the protection is only
-+ * against code executing on this cpu *not* from access by
-+ * other cpus.
-+ */
-+ if (unlikely(!this_cpu_cmpxchg_double(
-+ s->cpu_slab->freelist, s->cpu_slab->tid,
-+ object, tid,
-+ next_object, next_tid(tid)))) {
-+
-+ note_cmpxchg_failure("slab_alloc", s, tid);
-+ goto redo;
-+ }
-+ prefetch_freepointer(s, next_object);
-+ stat(s, ALLOC_FASTPATH);
-+ }
-+
-+ if (unlikely(gfpflags & __GFP_ZERO) && object)
-+ memset(object, 0, s->object_size);
-+
-+ slab_post_alloc_hook(s, gfpflags, object);
-+
-+ return object;
-+}
-+
-+static __always_inline void *slab_alloc(struct kmem_cache *s,
-+ gfp_t gfpflags, unsigned long addr)
-+{
-+ return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr);
-+}
-+
-+void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
-+{
-+ void *ret = slab_alloc(s, gfpflags, _RET_IP_);
-+
-+ trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
-+ s->size, gfpflags);
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL(kmem_cache_alloc);
-+
-+#ifdef CONFIG_TRACING
-+void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
-+{
-+ void *ret = slab_alloc(s, gfpflags, _RET_IP_);
-+ trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
-+ kasan_kmalloc(s, ret, size);
-+ return ret;
-+}
-+EXPORT_SYMBOL(kmem_cache_alloc_trace);
-+#endif
-+
-+#ifdef CONFIG_NUMA
-+void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
-+{
-+ void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
-+
-+ trace_kmem_cache_alloc_node(_RET_IP_, ret,
-+ s->object_size, s->size, gfpflags, node);
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL(kmem_cache_alloc_node);
-+
-+#ifdef CONFIG_TRACING
-+void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
-+ gfp_t gfpflags,
-+ int node, size_t size)
-+{
-+ void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
-+
-+ trace_kmalloc_node(_RET_IP_, ret,
-+ size, s->size, gfpflags, node);
-+
-+ kasan_kmalloc(s, ret, size);
-+ return ret;
-+}
-+EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
-+#endif
-+#endif
-+
-+/*
-+ * Slow path handling. This may still be called frequently since objects
-+ * have a longer lifetime than the cpu slabs in most processing loads.
-+ *
-+ * So we still attempt to reduce cache line usage. Just take the slab
-+ * lock and free the item. If there is no additional partial page
-+ * handling required then we can return immediately.
-+ */
-+static void __slab_free(struct kmem_cache *s, struct page *page,
-+ void *x, unsigned long addr)
-+{
-+ void *prior;
-+ void **object = (void *)x;
-+ int was_frozen;
-+ struct page new;
-+ unsigned long counters;
-+ struct kmem_cache_node *n = NULL;
-+ unsigned long uninitialized_var(flags);
-+
-+ stat(s, FREE_SLOWPATH);
-+
-+ if (kmem_cache_debug(s) &&
-+ !(n = free_debug_processing(s, page, x, addr, &flags)))
-+ return;
-+
-+ do {
-+ if (unlikely(n)) {
-+ spin_unlock_irqrestore(&n->list_lock, flags);
-+ n = NULL;
-+ }
-+ prior = page->freelist;
-+ counters = page->counters;
-+ set_freepointer(s, object, prior);
-+ new.counters = counters;
-+ was_frozen = new.frozen;
-+ new.inuse--;
-+ if ((!new.inuse || !prior) && !was_frozen) {
-+
-+ if (kmem_cache_has_cpu_partial(s) && !prior) {
-+
-+ /*
-+ * Slab was on no list before and will be
-+ * partially empty
-+ * We can defer the list move and instead
-+ * freeze it.
-+ */
-+ new.frozen = 1;
-+
-+ } else { /* Needs to be taken off a list */
-+
-+ n = get_node(s, page_to_nid(page));
-+ /*
-+ * Speculatively acquire the list_lock.
-+ * If the cmpxchg does not succeed then we may
-+ * drop the list_lock without any processing.
-+ *
-+ * Otherwise the list_lock will synchronize with
-+ * other processors updating the list of slabs.
-+ */
-+ spin_lock_irqsave(&n->list_lock, flags);
-+
-+ }
-+ }
-+
-+ } while (!cmpxchg_double_slab(s, page,
-+ prior, counters,
-+ object, new.counters,
-+ "__slab_free"));
-+
-+ if (likely(!n)) {
-+
-+ /*
-+ * If we just froze the page then put it onto the
-+ * per cpu partial list.
-+ */
-+ if (new.frozen && !was_frozen) {
-+ put_cpu_partial(s, page, 1);
-+ stat(s, CPU_PARTIAL_FREE);
-+ }
-+ /*
-+ * The list lock was not taken therefore no list
-+ * activity can be necessary.
-+ */
-+ if (was_frozen)
-+ stat(s, FREE_FROZEN);
-+ return;
-+ }
-+
-+ if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
-+ goto slab_empty;
-+
-+ /*
-+ * Objects left in the slab. If it was not on the partial list before
-+ * then add it.
-+ */
-+ if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
-+ if (kmem_cache_debug(s))
-+ remove_full(s, n, page);
-+ add_partial(n, page, DEACTIVATE_TO_TAIL);
-+ stat(s, FREE_ADD_PARTIAL);
-+ }
-+ spin_unlock_irqrestore(&n->list_lock, flags);
-+ return;
-+
-+slab_empty:
-+ if (prior) {
-+ /*
-+ * Slab on the partial list.
-+ */
-+ remove_partial(n, page);
-+ stat(s, FREE_REMOVE_PARTIAL);
-+ } else {
-+ /* Slab must be on the full list */
-+ remove_full(s, n, page);
-+ }
-+
-+ spin_unlock_irqrestore(&n->list_lock, flags);
-+ stat(s, FREE_SLAB);
-+ discard_slab(s, page);
-+}
-+
-+/*
-+ * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
-+ * can perform fastpath freeing without additional function calls.
-+ *
-+ * The fastpath is only possible if we are freeing to the current cpu slab
-+ * of this processor. This typically the case if we have just allocated
-+ * the item before.
-+ *
-+ * If fastpath is not possible then fall back to __slab_free where we deal
-+ * with all sorts of special processing.
-+ */
-+static __always_inline void slab_free(struct kmem_cache *s,
-+ struct page *page, void *x, unsigned long addr)
-+{
-+ void **object = (void *)x;
-+ struct kmem_cache_cpu *c;
-+ unsigned long tid;
-+
-+ slab_free_hook(s, x);
-+
-+redo:
-+ /*
-+ * Determine the currently cpus per cpu slab.
-+ * The cpu may change afterward. However that does not matter since
-+ * data is retrieved via this pointer. If we are on the same cpu
-+ * during the cmpxchg then the free will succedd.
-+ */
-+ do {
-+ tid = this_cpu_read(s->cpu_slab->tid);
-+ c = raw_cpu_ptr(s->cpu_slab);
-+ } while (IS_ENABLED(CONFIG_PREEMPT) &&
-+ unlikely(tid != READ_ONCE(c->tid)));
-+
-+ /* Same with comment on barrier() in slab_alloc_node() */
-+ barrier();
-+
-+ if (likely(page == c->page)) {
-+ set_freepointer(s, object, c->freelist);
-+
-+ if (unlikely(!this_cpu_cmpxchg_double(
-+ s->cpu_slab->freelist, s->cpu_slab->tid,
-+ c->freelist, tid,
-+ object, next_tid(tid)))) {
-+
-+ note_cmpxchg_failure("slab_free", s, tid);
-+ goto redo;
-+ }
-+ stat(s, FREE_FASTPATH);
-+ } else
-+ __slab_free(s, page, x, addr);
-+
-+}
-+
-+void kmem_cache_free(struct kmem_cache *s, void *x)
-+{
-+ s = cache_from_obj(s, x);
-+ if (!s)
-+ return;
-+ slab_free(s, virt_to_head_page(x), x, _RET_IP_);
-+ trace_kmem_cache_free(_RET_IP_, x);
-+}
-+EXPORT_SYMBOL(kmem_cache_free);
-+
-+/*
-+ * Object placement in a slab is made very easy because we always start at
-+ * offset 0. If we tune the size of the object to the alignment then we can
-+ * get the required alignment by putting one properly sized object after
-+ * another.
-+ *
-+ * Notice that the allocation order determines the sizes of the per cpu
-+ * caches. Each processor has always one slab available for allocations.
-+ * Increasing the allocation order reduces the number of times that slabs
-+ * must be moved on and off the partial lists and is therefore a factor in
-+ * locking overhead.
-+ */
-+
-+/*
-+ * Mininum / Maximum order of slab pages. This influences locking overhead
-+ * and slab fragmentation. A higher order reduces the number of partial slabs
-+ * and increases the number of allocations possible without having to
-+ * take the list_lock.
-+ */
-+static int slub_min_order;
-+static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
-+static int slub_min_objects;
-+
-+/*
-+ * Calculate the order of allocation given an slab object size.
-+ *
-+ * The order of allocation has significant impact on performance and other
-+ * system components. Generally order 0 allocations should be preferred since
-+ * order 0 does not cause fragmentation in the page allocator. Larger objects
-+ * be problematic to put into order 0 slabs because there may be too much
-+ * unused space left. We go to a higher order if more than 1/16th of the slab
-+ * would be wasted.
-+ *
-+ * In order to reach satisfactory performance we must ensure that a minimum
-+ * number of objects is in one slab. Otherwise we may generate too much
-+ * activity on the partial lists which requires taking the list_lock. This is
-+ * less a concern for large slabs though which are rarely used.
-+ *
-+ * slub_max_order specifies the order where we begin to stop considering the
-+ * number of objects in a slab as critical. If we reach slub_max_order then
-+ * we try to keep the page order as low as possible. So we accept more waste
-+ * of space in favor of a small page order.
-+ *
-+ * Higher order allocations also allow the placement of more objects in a
-+ * slab and thereby reduce object handling overhead. If the user has
-+ * requested a higher mininum order then we start with that one instead of
-+ * the smallest order which will fit the object.
-+ */
-+static inline int slab_order(int size, int min_objects,
-+ int max_order, int fract_leftover, int reserved)
-+{
-+ int order;
-+ int rem;
-+ int min_order = slub_min_order;
-+
-+ if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
-+ return get_order(size * MAX_OBJS_PER_PAGE) - 1;
-+
-+ for (order = max(min_order,
-+ fls(min_objects * size - 1) - PAGE_SHIFT);
-+ order <= max_order; order++) {
-+
-+ unsigned long slab_size = PAGE_SIZE << order;
-+
-+ if (slab_size < min_objects * size + reserved)
-+ continue;
-+
-+ rem = (slab_size - reserved) % size;
-+
-+ if (rem <= slab_size / fract_leftover)
-+ break;
-+
-+ }
-+
-+ return order;
-+}
-+
-+static inline int calculate_order(int size, int reserved)
-+{
-+ int order;
-+ int min_objects;
-+ int fraction;
-+ int max_objects;
-+
-+ /*
-+ * Attempt to find best configuration for a slab. This
-+ * works by first attempting to generate a layout with
-+ * the best configuration and backing off gradually.
-+ *
-+ * First we reduce the acceptable waste in a slab. Then
-+ * we reduce the minimum objects required in a slab.
-+ */
-+ min_objects = slub_min_objects;
-+ if (!min_objects)
-+ min_objects = 4 * (fls(nr_cpu_ids) + 1);
-+ max_objects = order_objects(slub_max_order, size, reserved);
-+ min_objects = min(min_objects, max_objects);
-+
-+ while (min_objects > 1) {
-+ fraction = 16;
-+ while (fraction >= 4) {
-+ order = slab_order(size, min_objects,
-+ slub_max_order, fraction, reserved);
-+ if (order <= slub_max_order)
-+ return order;
-+ fraction /= 2;
-+ }
-+ min_objects--;
-+ }
-+
-+ /*
-+ * We were unable to place multiple objects in a slab. Now
-+ * lets see if we can place a single object there.
-+ */
-+ order = slab_order(size, 1, slub_max_order, 1, reserved);
-+ if (order <= slub_max_order)
-+ return order;
-+
-+ /*
-+ * Doh this slab cannot be placed using slub_max_order.
-+ */
-+ order = slab_order(size, 1, MAX_ORDER, 1, reserved);
-+ if (order < MAX_ORDER)
-+ return order;
-+ return -ENOSYS;
-+}
-+
-+static void
-+init_kmem_cache_node(struct kmem_cache_node *n)
-+{
-+ n->nr_partial = 0;
-+ spin_lock_init(&n->list_lock);
-+ INIT_LIST_HEAD(&n->partial);
-+#ifdef CONFIG_SLUB_DEBUG
-+ atomic_long_set(&n->nr_slabs, 0);
-+ atomic_long_set(&n->total_objects, 0);
-+ INIT_LIST_HEAD(&n->full);
-+#endif
-+}
-+
-+static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
-+{
-+ BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
-+ KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
-+
-+ /*
-+ * Must align to double word boundary for the double cmpxchg
-+ * instructions to work; see __pcpu_double_call_return_bool().
-+ */
-+ s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
-+ 2 * sizeof(void *));
-+
-+ if (!s->cpu_slab)
-+ return 0;
-+
-+ init_kmem_cache_cpus(s);
-+
-+ return 1;
-+}
-+
-+static struct kmem_cache *kmem_cache_node;
-+
-+/*
-+ * No kmalloc_node yet so do it by hand. We know that this is the first
-+ * slab on the node for this slabcache. There are no concurrent accesses
-+ * possible.
-+ *
-+ * Note that this function only works on the kmem_cache_node
-+ * when allocating for the kmem_cache_node. This is used for bootstrapping
-+ * memory on a fresh node that has no slab structures yet.
-+ */
-+static void early_kmem_cache_node_alloc(int node)
-+{
-+ struct page *page;
-+ struct kmem_cache_node *n;
-+
-+ BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
-+
-+ page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
-+
-+ BUG_ON(!page);
-+ if (page_to_nid(page) != node) {
-+ pr_err("SLUB: Unable to allocate memory from node %d\n", node);
-+ pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
-+ }
-+
-+ n = page->freelist;
-+ BUG_ON(!n);
-+ page->freelist = get_freepointer(kmem_cache_node, n);
-+ page->inuse = 1;
-+ page->frozen = 0;
-+ kmem_cache_node->node[node] = n;
-+#ifdef CONFIG_SLUB_DEBUG
-+ init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
-+ init_tracking(kmem_cache_node, n);
-+#endif
-+ kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node));
-+ init_kmem_cache_node(n);
-+ inc_slabs_node(kmem_cache_node, node, page->objects);
-+
-+ /*
-+ * No locks need to be taken here as it has just been
-+ * initialized and there is no concurrent access.
-+ */
-+ __add_partial(n, page, DEACTIVATE_TO_HEAD);
-+}
-+
-+static void free_kmem_cache_nodes(struct kmem_cache *s)
-+{
-+ int node;
-+ struct kmem_cache_node *n;
-+
-+ for_each_kmem_cache_node(s, node, n) {
-+ kmem_cache_free(kmem_cache_node, n);
-+ s->node[node] = NULL;
-+ }
-+}
-+
-+static int init_kmem_cache_nodes(struct kmem_cache *s)
-+{
-+ int node;
-+
-+ for_each_node_state(node, N_NORMAL_MEMORY) {
-+ struct kmem_cache_node *n;
-+
-+ if (slab_state == DOWN) {
-+ early_kmem_cache_node_alloc(node);
-+ continue;
-+ }
-+ n = kmem_cache_alloc_node(kmem_cache_node,
-+ GFP_KERNEL, node);
-+
-+ if (!n) {
-+ free_kmem_cache_nodes(s);
-+ return 0;
-+ }
-+
-+ s->node[node] = n;
-+ init_kmem_cache_node(n);
-+ }
-+ return 1;
-+}
-+
-+static void set_min_partial(struct kmem_cache *s, unsigned long min)
-+{
-+ if (min < MIN_PARTIAL)
-+ min = MIN_PARTIAL;
-+ else if (min > MAX_PARTIAL)
-+ min = MAX_PARTIAL;
-+ s->min_partial = min;
-+}
-+
-+/*
-+ * calculate_sizes() determines the order and the distribution of data within
-+ * a slab object.
-+ */
-+static int calculate_sizes(struct kmem_cache *s, int forced_order)
-+{
-+ unsigned long flags = s->flags;
-+ unsigned long size = s->object_size;
-+ int order;
-+
-+ /*
-+ * Round up object size to the next word boundary. We can only
-+ * place the free pointer at word boundaries and this determines
-+ * the possible location of the free pointer.
-+ */
-+ size = ALIGN(size, sizeof(void *));
-+
-+#ifdef CONFIG_SLUB_DEBUG
-+ /*
-+ * Determine if we can poison the object itself. If the user of
-+ * the slab may touch the object after free or before allocation
-+ * then we should never poison the object itself.
-+ */
-+ if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
-+ !s->ctor)
-+ s->flags |= __OBJECT_POISON;
-+ else
-+ s->flags &= ~__OBJECT_POISON;
-+
-+
-+ /*
-+ * If we are Redzoning then check if there is some space between the
-+ * end of the object and the free pointer. If not then add an
-+ * additional word to have some bytes to store Redzone information.
-+ */
-+ if ((flags & SLAB_RED_ZONE) && size == s->object_size)
-+ size += sizeof(void *);
-+#endif
-+
-+ /*
-+ * With that we have determined the number of bytes in actual use
-+ * by the object. This is the potential offset to the free pointer.
-+ */
-+ s->inuse = size;
-+
-+ if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
-+ s->ctor)) {
-+ /*
-+ * Relocate free pointer after the object if it is not
-+ * permitted to overwrite the first word of the object on
-+ * kmem_cache_free.
-+ *
-+ * This is the case if we do RCU, have a constructor or
-+ * destructor or are poisoning the objects.
-+ */
-+ s->offset = size;
-+ size += sizeof(void *);
-+ }
-+
-+#ifdef CONFIG_SLUB_DEBUG
-+ if (flags & SLAB_STORE_USER)
-+ /*
-+ * Need to store information about allocs and frees after
-+ * the object.
-+ */
-+ size += 2 * sizeof(struct track);
-+
-+ if (flags & SLAB_RED_ZONE)
-+ /*
-+ * Add some empty padding so that we can catch
-+ * overwrites from earlier objects rather than let
-+ * tracking information or the free pointer be
-+ * corrupted if a user writes before the start
-+ * of the object.
-+ */
-+ size += sizeof(void *);
-+#endif
-+
-+ /*
-+ * SLUB stores one object immediately after another beginning from
-+ * offset 0. In order to align the objects we have to simply size
-+ * each object to conform to the alignment.
-+ */
-+ size = ALIGN(size, s->align);
-+ s->size = size;
-+ if (forced_order >= 0)
-+ order = forced_order;
-+ else
-+ order = calculate_order(size, s->reserved);
-+
-+ if (order < 0)
-+ return 0;
-+
-+ s->allocflags = 0;
-+ if (order)
-+ s->allocflags |= __GFP_COMP;
-+
-+ if (s->flags & SLAB_CACHE_DMA)
-+ s->allocflags |= GFP_DMA;
-+
-+ if (s->flags & SLAB_RECLAIM_ACCOUNT)
-+ s->allocflags |= __GFP_RECLAIMABLE;
-+
-+ /*
-+ * Determine the number of objects per slab
-+ */
-+ s->oo = oo_make(order, size, s->reserved);
-+ s->min = oo_make(get_order(size), size, s->reserved);
-+ if (oo_objects(s->oo) > oo_objects(s->max))
-+ s->max = s->oo;
-+
-+ return !!oo_objects(s->oo);
-+}
-+
-+static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
-+{
-+ s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor);
-+ s->reserved = 0;
-+
-+ if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
-+ s->reserved = sizeof(struct rcu_head);
-+
-+ if (!calculate_sizes(s, -1))
-+ goto error;
-+ if (disable_higher_order_debug) {
-+ /*
-+ * Disable debugging flags that store metadata if the min slab
-+ * order increased.
-+ */
-+ if (get_order(s->size) > get_order(s->object_size)) {
-+ s->flags &= ~DEBUG_METADATA_FLAGS;
-+ s->offset = 0;
-+ if (!calculate_sizes(s, -1))
-+ goto error;
-+ }
-+ }
-+
-+#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
-+ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
-+ if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
-+ /* Enable fast mode */
-+ s->flags |= __CMPXCHG_DOUBLE;
-+#endif
-+
-+ /*
-+ * The larger the object size is, the more pages we want on the partial
-+ * list to avoid pounding the page allocator excessively.
-+ */
-+ set_min_partial(s, ilog2(s->size) / 2);
-+
-+ /*
-+ * cpu_partial determined the maximum number of objects kept in the
-+ * per cpu partial lists of a processor.
-+ *
-+ * Per cpu partial lists mainly contain slabs that just have one
-+ * object freed. If they are used for allocation then they can be
-+ * filled up again with minimal effort. The slab will never hit the
-+ * per node partial lists and therefore no locking will be required.
-+ *
-+ * This setting also determines
-+ *
-+ * A) The number of objects from per cpu partial slabs dumped to the
-+ * per node list when we reach the limit.
-+ * B) The number of objects in cpu partial slabs to extract from the
-+ * per node list when we run out of per cpu objects. We only fetch
-+ * 50% to keep some capacity around for frees.
-+ */
-+ if (!kmem_cache_has_cpu_partial(s))
-+ s->cpu_partial = 0;
-+ else if (s->size >= PAGE_SIZE)
-+ s->cpu_partial = 2;
-+ else if (s->size >= 1024)
-+ s->cpu_partial = 6;
-+ else if (s->size >= 256)
-+ s->cpu_partial = 13;
-+ else
-+ s->cpu_partial = 30;
-+
-+#ifdef CONFIG_NUMA
-+ s->remote_node_defrag_ratio = 1000;
-+#endif
-+ if (!init_kmem_cache_nodes(s))
-+ goto error;
-+
-+ if (alloc_kmem_cache_cpus(s))
-+ return 0;
-+
-+ free_kmem_cache_nodes(s);
-+error:
-+ if (flags & SLAB_PANIC)
-+ panic("Cannot create slab %s size=%lu realsize=%u "
-+ "order=%u offset=%u flags=%lx\n",
-+ s->name, (unsigned long)s->size, s->size,
-+ oo_order(s->oo), s->offset, flags);
-+ return -EINVAL;
-+}
-+
-+static void list_slab_objects(struct kmem_cache *s, struct page *page,
-+ const char *text)
-+{
-+#ifdef CONFIG_SLUB_DEBUG
-+ void *addr = page_address(page);
-+ void *p;
-+ unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
-+ sizeof(long), GFP_ATOMIC);
-+ if (!map)
-+ return;
-+ slab_err(s, page, text, s->name);
-+ slab_lock(page);
-+
-+ get_map(s, page, map);
-+ for_each_object(p, s, addr, page->objects) {
-+
-+ if (!test_bit(slab_index(p, s, addr), map)) {
-+ pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr);
-+ print_tracking(s, p);
-+ }
-+ }
-+ slab_unlock(page);
-+ kfree(map);
-+#endif
-+}
-+
-+/*
-+ * Attempt to free all partial slabs on a node.
-+ * This is called from kmem_cache_close(). We must be the last thread
-+ * using the cache and therefore we do not need to lock anymore.
-+ */
-+static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
-+{
-+ struct page *page, *h;
-+
-+ list_for_each_entry_safe(page, h, &n->partial, lru) {
-+ if (!page->inuse) {
-+ __remove_partial(n, page);
-+ discard_slab(s, page);
-+ } else {
-+ list_slab_objects(s, page,
-+ "Objects remaining in %s on kmem_cache_close()");
-+ }
-+ }
-+}
-+
-+/*
-+ * Release all resources used by a slab cache.
-+ */
-+static inline int kmem_cache_close(struct kmem_cache *s)
-+{
-+ int node;
-+ struct kmem_cache_node *n;
-+
-+ flush_all(s);
-+ /* Attempt to free all objects */
-+ for_each_kmem_cache_node(s, node, n) {
-+ free_partial(s, n);
-+ if (n->nr_partial || slabs_node(s, node))
-+ return 1;
-+ }
-+ free_percpu(s->cpu_slab);
-+ free_kmem_cache_nodes(s);
-+ return 0;
-+}
-+
-+int __kmem_cache_shutdown(struct kmem_cache *s)
-+{
-+ return kmem_cache_close(s);
-+}
-+
-+/********************************************************************
-+ * Kmalloc subsystem
-+ *******************************************************************/
-+
-+static int __init setup_slub_min_order(char *str)
-+{
-+ get_option(&str, &slub_min_order);
-+
-+ return 1;
-+}
-+
-+__setup("slub_min_order=", setup_slub_min_order);
-+
-+static int __init setup_slub_max_order(char *str)
-+{
-+ get_option(&str, &slub_max_order);
-+ slub_max_order = min(slub_max_order, MAX_ORDER - 1);
-+
-+ return 1;
-+}
-+
-+__setup("slub_max_order=", setup_slub_max_order);
-+
-+static int __init setup_slub_min_objects(char *str)
-+{
-+ get_option(&str, &slub_min_objects);
-+
-+ return 1;
-+}
-+
-+__setup("slub_min_objects=", setup_slub_min_objects);
-+
-+void *__kmalloc(size_t size, gfp_t flags)
-+{
-+ struct kmem_cache *s;
-+ void *ret;
-+
-+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
-+ return kmalloc_large(size, flags);
-+
-+ s = kmalloc_slab(size, flags);
-+
-+ if (unlikely(ZERO_OR_NULL_PTR(s)))
-+ return s;
-+
-+ ret = slab_alloc(s, flags, _RET_IP_);
-+
-+ trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
-+
-+ kasan_kmalloc(s, ret, size);
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL(__kmalloc);
-+
-+#ifdef CONFIG_NUMA
-+static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
-+{
-+ struct page *page;
-+ void *ptr = NULL;
-+
-+ flags |= __GFP_COMP | __GFP_NOTRACK;
-+ page = alloc_kmem_pages_node(node, flags, get_order(size));
-+ if (page)
-+ ptr = page_address(page);
-+
-+ kmalloc_large_node_hook(ptr, size, flags);
-+ return ptr;
-+}
-+
-+void *__kmalloc_node(size_t size, gfp_t flags, int node)
-+{
-+ struct kmem_cache *s;
-+ void *ret;
-+
-+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
-+ ret = kmalloc_large_node(size, flags, node);
-+
-+ trace_kmalloc_node(_RET_IP_, ret,
-+ size, PAGE_SIZE << get_order(size),
-+ flags, node);
-+
-+ return ret;
-+ }
-+
-+ s = kmalloc_slab(size, flags);
-+
-+ if (unlikely(ZERO_OR_NULL_PTR(s)))
-+ return s;
-+
-+ ret = slab_alloc_node(s, flags, node, _RET_IP_);
-+
-+ trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
-+
-+ kasan_kmalloc(s, ret, size);
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL(__kmalloc_node);
-+#endif
-+
-+static size_t __ksize(const void *object)
-+{
-+ struct page *page;
-+
-+ if (unlikely(object == ZERO_SIZE_PTR))
-+ return 0;
-+
-+ page = virt_to_head_page(object);
-+
-+ if (unlikely(!PageSlab(page))) {
-+ WARN_ON(!PageCompound(page));
-+ return PAGE_SIZE << compound_order(page);
-+ }
-+
-+ return slab_ksize(page->slab_cache);
-+}
-+
-+size_t ksize(const void *object)
-+{
-+ size_t size = __ksize(object);
-+ /* We assume that ksize callers could use whole allocated area,
-+ so we need unpoison this area. */
-+ kasan_krealloc(object, size);
-+ return size;
-+}
-+EXPORT_SYMBOL(ksize);
-+
-+void kfree(const void *x)
-+{
-+ struct page *page;
-+ void *object = (void *)x;
-+
-+ trace_kfree(_RET_IP_, x);
-+
-+ if (unlikely(ZERO_OR_NULL_PTR(x)))
-+ return;
-+
-+ page = virt_to_head_page(x);
-+ if (unlikely(!PageSlab(page))) {
-+ BUG_ON(!PageCompound(page));
-+ kfree_hook(x);
-+ __free_kmem_pages(page, compound_order(page));
-+ return;
-+ }
-+ slab_free(page->slab_cache, page, object, _RET_IP_);
-+}
-+EXPORT_SYMBOL(kfree);
-+
-+#define SHRINK_PROMOTE_MAX 32
-+
-+/*
-+ * kmem_cache_shrink discards empty slabs and promotes the slabs filled
-+ * up most to the head of the partial lists. New allocations will then
-+ * fill those up and thus they can be removed from the partial lists.
-+ *
-+ * The slabs with the least items are placed last. This results in them
-+ * being allocated from last increasing the chance that the last objects
-+ * are freed in them.
-+ */
-+int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate)
-+{
-+ int node;
-+ int i;
-+ struct kmem_cache_node *n;
-+ struct page *page;
-+ struct page *t;
-+ struct list_head discard;
-+ struct list_head promote[SHRINK_PROMOTE_MAX];
-+ unsigned long flags;
-+ int ret = 0;
-+
-+ if (deactivate) {
-+ /*
-+ * Disable empty slabs caching. Used to avoid pinning offline
-+ * memory cgroups by kmem pages that can be freed.
-+ */
-+ s->cpu_partial = 0;
-+ s->min_partial = 0;
-+
-+ /*
-+ * s->cpu_partial is checked locklessly (see put_cpu_partial),
-+ * so we have to make sure the change is visible.
-+ */
-+ kick_all_cpus_sync();
-+ }
-+
-+ flush_all(s);
-+ for_each_kmem_cache_node(s, node, n) {
-+ INIT_LIST_HEAD(&discard);
-+ for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
-+ INIT_LIST_HEAD(promote + i);
-+
-+ spin_lock_irqsave(&n->list_lock, flags);
-+
-+ /*
-+ * Build lists of slabs to discard or promote.
-+ *
-+ * Note that concurrent frees may occur while we hold the
-+ * list_lock. page->inuse here is the upper limit.
-+ */
-+ list_for_each_entry_safe(page, t, &n->partial, lru) {
-+ int free = page->objects - page->inuse;
-+
-+ /* Do not reread page->inuse */
-+ barrier();
-+
-+ /* We do not keep full slabs on the list */
-+ BUG_ON(free <= 0);
-+
-+ if (free == page->objects) {
-+ list_move(&page->lru, &discard);
-+ n->nr_partial--;
-+ } else if (free <= SHRINK_PROMOTE_MAX)
-+ list_move(&page->lru, promote + free - 1);
-+ }
-+
-+ /*
-+ * Promote the slabs filled up most to the head of the
-+ * partial list.
-+ */
-+ for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
-+ list_splice(promote + i, &n->partial);
-+
-+ spin_unlock_irqrestore(&n->list_lock, flags);
-+
-+ /* Release empty slabs */
-+ list_for_each_entry_safe(page, t, &discard, lru)
-+ discard_slab(s, page);
-+
-+ if (slabs_node(s, node))
-+ ret = 1;
-+ }
-+
-+ return ret;
-+}
-+
-+static int slab_mem_going_offline_callback(void *arg)
-+{
-+ struct kmem_cache *s;
-+
-+ mutex_lock(&slab_mutex);
-+ list_for_each_entry(s, &slab_caches, list)
-+ __kmem_cache_shrink(s, false);
-+ mutex_unlock(&slab_mutex);
-+
-+ return 0;
-+}
-+
-+static void slab_mem_offline_callback(void *arg)
-+{
-+ struct kmem_cache_node *n;
-+ struct kmem_cache *s;
-+ struct memory_notify *marg = arg;
-+ int offline_node;
-+
-+ offline_node = marg->status_change_nid_normal;
-+
-+ /*
-+ * If the node still has available memory. we need kmem_cache_node
-+ * for it yet.
-+ */
-+ if (offline_node < 0)
-+ return;
-+
-+ mutex_lock(&slab_mutex);
-+ list_for_each_entry(s, &slab_caches, list) {
-+ n = get_node(s, offline_node);
-+ if (n) {
-+ /*
-+ * if n->nr_slabs > 0, slabs still exist on the node
-+ * that is going down. We were unable to free them,
-+ * and offline_pages() function shouldn't call this
-+ * callback. So, we must fail.
-+ */
-+ BUG_ON(slabs_node(s, offline_node));
-+
-+ s->node[offline_node] = NULL;
-+ kmem_cache_free(kmem_cache_node, n);
-+ }
-+ }
-+ mutex_unlock(&slab_mutex);
-+}
-+
-+static int slab_mem_going_online_callback(void *arg)
-+{
-+ struct kmem_cache_node *n;
-+ struct kmem_cache *s;
-+ struct memory_notify *marg = arg;
-+ int nid = marg->status_change_nid_normal;
-+ int ret = 0;
-+
-+ /*
-+ * If the node's memory is already available, then kmem_cache_node is
-+ * already created. Nothing to do.
-+ */
-+ if (nid < 0)
-+ return 0;
-+
-+ /*
-+ * We are bringing a node online. No memory is available yet. We must
-+ * allocate a kmem_cache_node structure in order to bring the node
-+ * online.
-+ */
-+ mutex_lock(&slab_mutex);
-+ list_for_each_entry(s, &slab_caches, list) {
-+ /*
-+ * XXX: kmem_cache_alloc_node will fallback to other nodes
-+ * since memory is not yet available from the node that
-+ * is brought up.
-+ */
-+ n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
-+ if (!n) {
-+ ret = -ENOMEM;
-+ goto out;
-+ }
-+ init_kmem_cache_node(n);
-+ s->node[nid] = n;
-+ }
-+out:
-+ mutex_unlock(&slab_mutex);
-+ return ret;
-+}
-+
-+static int slab_memory_callback(struct notifier_block *self,
-+ unsigned long action, void *arg)
-+{
-+ int ret = 0;
-+
-+ switch (action) {
-+ case MEM_GOING_ONLINE:
-+ ret = slab_mem_going_online_callback(arg);
-+ break;
-+ case MEM_GOING_OFFLINE:
-+ ret = slab_mem_going_offline_callback(arg);
-+ break;
-+ case MEM_OFFLINE:
-+ case MEM_CANCEL_ONLINE:
-+ slab_mem_offline_callback(arg);
-+ break;
-+ case MEM_ONLINE:
-+ case MEM_CANCEL_OFFLINE:
-+ break;
-+ }
-+ if (ret)
-+ ret = notifier_from_errno(ret);
-+ else
-+ ret = NOTIFY_OK;
-+ return ret;
-+}
-+
-+static struct notifier_block slab_memory_callback_nb = {
-+ .notifier_call = slab_memory_callback,
-+ .priority = SLAB_CALLBACK_PRI,
-+};
-+
-+/********************************************************************
-+ * Basic setup of slabs
-+ *******************************************************************/
-+
-+/*
-+ * Used for early kmem_cache structures that were allocated using
-+ * the page allocator. Allocate them properly then fix up the pointers
-+ * that may be pointing to the wrong kmem_cache structure.
-+ */
-+
-+static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
-+{
-+ int node;
-+ struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-+ struct kmem_cache_node *n;
-+
-+ memcpy(s, static_cache, kmem_cache->object_size);
-+
-+ /*
-+ * This runs very early, and only the boot processor is supposed to be
-+ * up. Even if it weren't true, IRQs are not up so we couldn't fire
-+ * IPIs around.
-+ */
-+ __flush_cpu_slab(s, smp_processor_id());
-+ for_each_kmem_cache_node(s, node, n) {
-+ struct page *p;
-+
-+ list_for_each_entry(p, &n->partial, lru)
-+ p->slab_cache = s;
-+
-+#ifdef CONFIG_SLUB_DEBUG
-+ list_for_each_entry(p, &n->full, lru)
-+ p->slab_cache = s;
-+#endif
-+ }
-+ slab_init_memcg_params(s);
-+ list_add(&s->list, &slab_caches);
-+ return s;
-+}
-+
-+void __init kmem_cache_init(void)
-+{
-+ static __initdata struct kmem_cache boot_kmem_cache,
-+ boot_kmem_cache_node;
-+
-+ if (debug_guardpage_minorder())
-+ slub_max_order = 0;
-+
-+ kmem_cache_node = &boot_kmem_cache_node;
-+ kmem_cache = &boot_kmem_cache;
-+
-+ create_boot_cache(kmem_cache_node, "kmem_cache_node",
-+ sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
-+
-+ register_hotmemory_notifier(&slab_memory_callback_nb);
-+
-+ /* Able to allocate the per node structures */
-+ slab_state = PARTIAL;
-+
-+ create_boot_cache(kmem_cache, "kmem_cache",
-+ offsetof(struct kmem_cache, node) +
-+ nr_node_ids * sizeof(struct kmem_cache_node *),
-+ SLAB_HWCACHE_ALIGN);
-+
-+ kmem_cache = bootstrap(&boot_kmem_cache);
-+
-+ /*
-+ * Allocate kmem_cache_node properly from the kmem_cache slab.
-+ * kmem_cache_node is separately allocated so no need to
-+ * update any list pointers.
-+ */
-+ kmem_cache_node = bootstrap(&boot_kmem_cache_node);
-+
-+ /* Now we can use the kmem_cache to allocate kmalloc slabs */
-+ create_kmalloc_caches(0);
-+
-+#ifdef CONFIG_SMP
-+ register_cpu_notifier(&slab_notifier);
-+#endif
-+
-+ pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n",
-+ cache_line_size(),
-+ slub_min_order, slub_max_order, slub_min_objects,
-+ nr_cpu_ids, nr_node_ids);
-+}
-+
-+void __init kmem_cache_init_late(void)
-+{
-+}
-+
-+struct kmem_cache *
-+__kmem_cache_alias(const char *name, size_t size, size_t align,
-+ unsigned long flags, void (*ctor)(void *))
-+{
-+ struct kmem_cache *s, *c;
-+
-+ s = find_mergeable(size, align, flags, name, ctor);
-+ if (s) {
-+ s->refcount++;
-+
-+ /*
-+ * Adjust the object sizes so that we clear
-+ * the complete object on kzalloc.
-+ */
-+ s->object_size = max(s->object_size, (int)size);
-+ s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
-+
-+ for_each_memcg_cache(c, s) {
-+ c->object_size = s->object_size;
-+ c->inuse = max_t(int, c->inuse,
-+ ALIGN(size, sizeof(void *)));
-+ }
-+
-+ if (sysfs_slab_alias(s, name)) {
-+ s->refcount--;
-+ s = NULL;
-+ }
-+ }
-+
-+ return s;
-+}
-+
-+int __kmem_cache_create(struct kmem_cache *s, unsigned long flags)
-+{
-+ int err;
-+
-+ err = kmem_cache_open(s, flags);
-+ if (err)
-+ return err;
-+
-+ /* Mutex is not taken during early boot */
-+ if (slab_state <= UP)
-+ return 0;
-+
-+ memcg_propagate_slab_attrs(s);
-+ err = sysfs_slab_add(s);
-+ if (err)
-+ kmem_cache_close(s);
-+
-+ return err;
-+}
-+
-+#ifdef CONFIG_SMP
-+/*
-+ * Use the cpu notifier to insure that the cpu slabs are flushed when
-+ * necessary.
-+ */
-+static int slab_cpuup_callback(struct notifier_block *nfb,
-+ unsigned long action, void *hcpu)
-+{
-+ long cpu = (long)hcpu;
-+ struct kmem_cache *s;
-+ unsigned long flags;
-+
-+ switch (action) {
-+ case CPU_UP_CANCELED:
-+ case CPU_UP_CANCELED_FROZEN:
-+ case CPU_DEAD:
-+ case CPU_DEAD_FROZEN:
-+ mutex_lock(&slab_mutex);
-+ list_for_each_entry(s, &slab_caches, list) {
-+ local_irq_save(flags);
-+ __flush_cpu_slab(s, cpu);
-+ local_irq_restore(flags);
-+ }
-+ mutex_unlock(&slab_mutex);
-+ break;
-+ default:
-+ break;
-+ }
-+ return NOTIFY_OK;
-+}
-+
-+static struct notifier_block slab_notifier = {
-+ .notifier_call = slab_cpuup_callback
-+};
-+
-+#endif
-+
-+void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
-+{
-+ struct kmem_cache *s;
-+ void *ret;
-+
-+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
-+ return kmalloc_large(size, gfpflags);
-+
-+ s = kmalloc_slab(size, gfpflags);
-+
-+ if (unlikely(ZERO_OR_NULL_PTR(s)))
-+ return s;
-+
-+ ret = slab_alloc(s, gfpflags, caller);
-+
-+ /* Honor the call site pointer we received. */
-+ trace_kmalloc(caller, ret, size, s->size, gfpflags);
-+
-+ return ret;
-+}
-+
-+#ifdef CONFIG_NUMA
-+void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
-+ int node, unsigned long caller)
-+{
-+ struct kmem_cache *s;
-+ void *ret;
-+
-+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
-+ ret = kmalloc_large_node(size, gfpflags, node);
-+
-+ trace_kmalloc_node(caller, ret,
-+ size, PAGE_SIZE << get_order(size),
-+ gfpflags, node);
-+
-+ return ret;
-+ }
-+
-+ s = kmalloc_slab(size, gfpflags);
-+
-+ if (unlikely(ZERO_OR_NULL_PTR(s)))
-+ return s;
-+
-+ ret = slab_alloc_node(s, gfpflags, node, caller);
-+
-+ /* Honor the call site pointer we received. */
-+ trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
-+
-+ return ret;
-+}
-+#endif
-+
-+#ifdef CONFIG_SYSFS
-+static int count_inuse(struct page *page)
-+{
-+ return page->inuse;
-+}
-+
-+static int count_total(struct page *page)
-+{
-+ return page->objects;
-+}
-+#endif
-+
-+#ifdef CONFIG_SLUB_DEBUG
-+static int validate_slab(struct kmem_cache *s, struct page *page,
-+ unsigned long *map)
-+{
-+ void *p;
-+ void *addr = page_address(page);
-+
-+ if (!check_slab(s, page) ||
-+ !on_freelist(s, page, NULL))
-+ return 0;
-+
-+ /* Now we know that a valid freelist exists */
-+ bitmap_zero(map, page->objects);
-+
-+ get_map(s, page, map);
-+ for_each_object(p, s, addr, page->objects) {
-+ if (test_bit(slab_index(p, s, addr), map))
-+ if (!check_object(s, page, p, SLUB_RED_INACTIVE))
-+ return 0;
-+ }
-+
-+ for_each_object(p, s, addr, page->objects)
-+ if (!test_bit(slab_index(p, s, addr), map))
-+ if (!check_object(s, page, p, SLUB_RED_ACTIVE))
-+ return 0;
-+ return 1;
-+}
-+
-+static void validate_slab_slab(struct kmem_cache *s, struct page *page,
-+ unsigned long *map)
-+{
-+ slab_lock(page);
-+ validate_slab(s, page, map);
-+ slab_unlock(page);
-+}
-+
-+static int validate_slab_node(struct kmem_cache *s,
-+ struct kmem_cache_node *n, unsigned long *map)
-+{
-+ unsigned long count = 0;
-+ struct page *page;
-+ unsigned long flags;
-+
-+ spin_lock_irqsave(&n->list_lock, flags);
-+
-+ list_for_each_entry(page, &n->partial, lru) {
-+ validate_slab_slab(s, page, map);
-+ count++;
-+ }
-+ if (count != n->nr_partial)
-+ pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
-+ s->name, count, n->nr_partial);
-+
-+ if (!(s->flags & SLAB_STORE_USER))
-+ goto out;
-+
-+ list_for_each_entry(page, &n->full, lru) {
-+ validate_slab_slab(s, page, map);
-+ count++;
-+ }
-+ if (count != atomic_long_read(&n->nr_slabs))
-+ pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
-+ s->name, count, atomic_long_read(&n->nr_slabs));
-+
-+out:
-+ spin_unlock_irqrestore(&n->list_lock, flags);
-+ return count;
-+}
-+
-+static long validate_slab_cache(struct kmem_cache *s)
-+{
-+ int node;
-+ unsigned long count = 0;
-+ unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
-+ sizeof(unsigned long), GFP_KERNEL);
-+ struct kmem_cache_node *n;
-+
-+ if (!map)
-+ return -ENOMEM;
-+
-+ flush_all(s);
-+ for_each_kmem_cache_node(s, node, n)
-+ count += validate_slab_node(s, n, map);
-+ kfree(map);
-+ return count;
-+}
-+/*
-+ * Generate lists of code addresses where slabcache objects are allocated
-+ * and freed.
-+ */
-+
-+struct location {
-+ unsigned long count;
-+ unsigned long addr;
-+ long long sum_time;
-+ long min_time;
-+ long max_time;
-+ long min_pid;
-+ long max_pid;
-+ DECLARE_BITMAP(cpus, NR_CPUS);
-+ nodemask_t nodes;
-+};
-+
-+struct loc_track {
-+ unsigned long max;
-+ unsigned long count;
-+ struct location *loc;
-+};
-+
-+static void free_loc_track(struct loc_track *t)
-+{
-+ if (t->max)
-+ free_pages((unsigned long)t->loc,
-+ get_order(sizeof(struct location) * t->max));
-+}
-+
-+static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
-+{
-+ struct location *l;
-+ int order;
-+
-+ order = get_order(sizeof(struct location) * max);
-+
-+ l = (void *)__get_free_pages(flags, order);
-+ if (!l)
-+ return 0;
-+
-+ if (t->count) {
-+ memcpy(l, t->loc, sizeof(struct location) * t->count);
-+ free_loc_track(t);
-+ }
-+ t->max = max;
-+ t->loc = l;
-+ return 1;
-+}
-+
-+static int add_location(struct loc_track *t, struct kmem_cache *s,
-+ const struct track *track)
-+{
-+ long start, end, pos;
-+ struct location *l;
-+ unsigned long caddr;
-+ unsigned long age = jiffies - track->when;
-+
-+ start = -1;
-+ end = t->count;
-+
-+ for ( ; ; ) {
-+ pos = start + (end - start + 1) / 2;
-+
-+ /*
-+ * There is nothing at "end". If we end up there
-+ * we need to add something to before end.
-+ */
-+ if (pos == end)
-+ break;
-+
-+ caddr = t->loc[pos].addr;
-+ if (track->addr == caddr) {
-+
-+ l = &t->loc[pos];
-+ l->count++;
-+ if (track->when) {
-+ l->sum_time += age;
-+ if (age < l->min_time)
-+ l->min_time = age;
-+ if (age > l->max_time)
-+ l->max_time = age;
-+
-+ if (track->pid < l->min_pid)
-+ l->min_pid = track->pid;
-+ if (track->pid > l->max_pid)
-+ l->max_pid = track->pid;
-+
-+ cpumask_set_cpu(track->cpu,
-+ to_cpumask(l->cpus));
-+ }
-+ node_set(page_to_nid(virt_to_page(track)), l->nodes);
-+ return 1;
-+ }
-+
-+ if (track->addr < caddr)
-+ end = pos;
-+ else
-+ start = pos;
-+ }
-+
-+ /*
-+ * Not found. Insert new tracking element.
-+ */
-+ if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
-+ return 0;
-+
-+ l = t->loc + pos;
-+ if (pos < t->count)
-+ memmove(l + 1, l,
-+ (t->count - pos) * sizeof(struct location));
-+ t->count++;
-+ l->count = 1;
-+ l->addr = track->addr;
-+ l->sum_time = age;
-+ l->min_time = age;
-+ l->max_time = age;
-+ l->min_pid = track->pid;
-+ l->max_pid = track->pid;
-+ cpumask_clear(to_cpumask(l->cpus));
-+ cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
-+ nodes_clear(l->nodes);
-+ node_set(page_to_nid(virt_to_page(track)), l->nodes);
-+ return 1;
-+}
-+
-+static void process_slab(struct loc_track *t, struct kmem_cache *s,
-+ struct page *page, enum track_item alloc,
-+ unsigned long *map)
-+{
-+ void *addr = page_address(page);
-+ void *p;
-+
-+ bitmap_zero(map, page->objects);
-+ get_map(s, page, map);
-+
-+ for_each_object(p, s, addr, page->objects)
-+ if (!test_bit(slab_index(p, s, addr), map))
-+ add_location(t, s, get_track(s, p, alloc));
-+}
-+
-+static int list_locations(struct kmem_cache *s, char *buf,
-+ enum track_item alloc)
-+{
-+ int len = 0;
-+ unsigned long i;
-+ struct loc_track t = { 0, 0, NULL };
-+ int node;
-+ unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
-+ sizeof(unsigned long), GFP_KERNEL);
-+ struct kmem_cache_node *n;
-+
-+ if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
-+ GFP_TEMPORARY)) {
-+ kfree(map);
-+ return sprintf(buf, "Out of memory\n");
-+ }
-+ /* Push back cpu slabs */
-+ flush_all(s);
-+
-+ for_each_kmem_cache_node(s, node, n) {
-+ unsigned long flags;
-+ struct page *page;
-+
-+ if (!atomic_long_read(&n->nr_slabs))
-+ continue;
-+
-+ spin_lock_irqsave(&n->list_lock, flags);
-+ list_for_each_entry(page, &n->partial, lru)
-+ process_slab(&t, s, page, alloc, map);
-+ list_for_each_entry(page, &n->full, lru)
-+ process_slab(&t, s, page, alloc, map);
-+ spin_unlock_irqrestore(&n->list_lock, flags);
-+ }
-+
-+ for (i = 0; i < t.count; i++) {
-+ struct location *l = &t.loc[i];
-+
-+ if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
-+ break;
-+ len += sprintf(buf + len, "%7ld ", l->count);
-+
-+ if (l->addr)
-+ len += sprintf(buf + len, "%pS", (void *)l->addr);
-+ else
-+ len += sprintf(buf + len, "<not-available>");
-+
-+ if (l->sum_time != l->min_time) {
-+ len += sprintf(buf + len, " age=%ld/%ld/%ld",
-+ l->min_time,
-+ (long)div_u64(l->sum_time, l->count),
-+ l->max_time);
-+ } else
-+ len += sprintf(buf + len, " age=%ld",
-+ l->min_time);
-+
-+ if (l->min_pid != l->max_pid)
-+ len += sprintf(buf + len, " pid=%ld-%ld",
-+ l->min_pid, l->max_pid);
-+ else
-+ len += sprintf(buf + len, " pid=%ld",
-+ l->min_pid);
-+
-+ if (num_online_cpus() > 1 &&
-+ !cpumask_empty(to_cpumask(l->cpus)) &&
-+ len < PAGE_SIZE - 60)
-+ len += scnprintf(buf + len, PAGE_SIZE - len - 50,
-+ " cpus=%*pbl",
-+ cpumask_pr_args(to_cpumask(l->cpus)));
-+
-+ if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
-+ len < PAGE_SIZE - 60)
-+ len += scnprintf(buf + len, PAGE_SIZE - len - 50,
-+ " nodes=%*pbl",
-+ nodemask_pr_args(&l->nodes));
-+
-+ len += sprintf(buf + len, "\n");
-+ }
-+
-+ free_loc_track(&t);
-+ kfree(map);
-+ if (!t.count)
-+ len += sprintf(buf, "No data\n");
-+ return len;
-+}
-+#endif
-+
-+#ifdef SLUB_RESILIENCY_TEST
-+static void __init resiliency_test(void)
-+{
-+ u8 *p;
-+
-+ BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
-+
-+ pr_err("SLUB resiliency testing\n");
-+ pr_err("-----------------------\n");
-+ pr_err("A. Corruption after allocation\n");
-+
-+ p = kzalloc(16, GFP_KERNEL);
-+ p[16] = 0x12;
-+ pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
-+ p + 16);
-+
-+ validate_slab_cache(kmalloc_caches[4]);
-+
-+ /* Hmmm... The next two are dangerous */
-+ p = kzalloc(32, GFP_KERNEL);
-+ p[32 + sizeof(void *)] = 0x34;
-+ pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
-+ p);
-+ pr_err("If allocated object is overwritten then not detectable\n\n");
-+
-+ validate_slab_cache(kmalloc_caches[5]);
-+ p = kzalloc(64, GFP_KERNEL);
-+ p += 64 + (get_cycles() & 0xff) * sizeof(void *);
-+ *p = 0x56;
-+ pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
-+ p);
-+ pr_err("If allocated object is overwritten then not detectable\n\n");
-+ validate_slab_cache(kmalloc_caches[6]);
-+
-+ pr_err("\nB. Corruption after free\n");
-+ p = kzalloc(128, GFP_KERNEL);
-+ kfree(p);
-+ *p = 0x78;
-+ pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
-+ validate_slab_cache(kmalloc_caches[7]);
-+
-+ p = kzalloc(256, GFP_KERNEL);
-+ kfree(p);
-+ p[50] = 0x9a;
-+ pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
-+ validate_slab_cache(kmalloc_caches[8]);
-+
-+ p = kzalloc(512, GFP_KERNEL);
-+ kfree(p);
-+ p[512] = 0xab;
-+ pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
-+ validate_slab_cache(kmalloc_caches[9]);
-+}
-+#else
-+#ifdef CONFIG_SYSFS
-+static void resiliency_test(void) {};
-+#endif
-+#endif
-+
-+#ifdef CONFIG_SYSFS
-+enum slab_stat_type {
-+ SL_ALL, /* All slabs */
-+ SL_PARTIAL, /* Only partially allocated slabs */
-+ SL_CPU, /* Only slabs used for cpu caches */
-+ SL_OBJECTS, /* Determine allocated objects not slabs */
-+ SL_TOTAL /* Determine object capacity not slabs */
-+};
-+
-+#define SO_ALL (1 << SL_ALL)
-+#define SO_PARTIAL (1 << SL_PARTIAL)
-+#define SO_CPU (1 << SL_CPU)
-+#define SO_OBJECTS (1 << SL_OBJECTS)
-+#define SO_TOTAL (1 << SL_TOTAL)
-+
-+static ssize_t show_slab_objects(struct kmem_cache *s,
-+ char *buf, unsigned long flags)
-+{
-+ unsigned long total = 0;
-+ int node;
-+ int x;
-+ unsigned long *nodes;
-+
-+ nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
-+ if (!nodes)
-+ return -ENOMEM;
-+
-+ if (flags & SO_CPU) {
-+ int cpu;
-+
-+ for_each_possible_cpu(cpu) {
-+ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
-+ cpu);
-+ int node;
-+ struct page *page;
-+
-+ page = READ_ONCE(c->page);
-+ if (!page)
-+ continue;
-+
-+ node = page_to_nid(page);
-+ if (flags & SO_TOTAL)
-+ x = page->objects;
-+ else if (flags & SO_OBJECTS)
-+ x = page->inuse;
-+ else
-+ x = 1;
-+
-+ total += x;
-+ nodes[node] += x;
-+
-+ page = READ_ONCE(c->partial);
-+ if (page) {
-+ node = page_to_nid(page);
-+ if (flags & SO_TOTAL)
-+ WARN_ON_ONCE(1);
-+ else if (flags & SO_OBJECTS)
-+ WARN_ON_ONCE(1);
-+ else
-+ x = page->pages;
-+ total += x;
-+ nodes[node] += x;
-+ }
-+ }
-+ }
-+
-+ get_online_mems();
-+#ifdef CONFIG_SLUB_DEBUG
-+ if (flags & SO_ALL) {
-+ struct kmem_cache_node *n;
-+
-+ for_each_kmem_cache_node(s, node, n) {
-+
-+ if (flags & SO_TOTAL)
-+ x = atomic_long_read(&n->total_objects);
-+ else if (flags & SO_OBJECTS)
-+ x = atomic_long_read(&n->total_objects) -
-+ count_partial(n, count_free);
-+ else
-+ x = atomic_long_read(&n->nr_slabs);
-+ total += x;
-+ nodes[node] += x;
-+ }
-+
-+ } else
-+#endif
-+ if (flags & SO_PARTIAL) {
-+ struct kmem_cache_node *n;
-+
-+ for_each_kmem_cache_node(s, node, n) {
-+ if (flags & SO_TOTAL)
-+ x = count_partial(n, count_total);
-+ else if (flags & SO_OBJECTS)
-+ x = count_partial(n, count_inuse);
-+ else
-+ x = n->nr_partial;
-+ total += x;
-+ nodes[node] += x;
-+ }
-+ }
-+ x = sprintf(buf, "%lu", total);
-+#ifdef CONFIG_NUMA
-+ for (node = 0; node < nr_node_ids; node++)
-+ if (nodes[node])
-+ x += sprintf(buf + x, " N%d=%lu",
-+ node, nodes[node]);
-+#endif
-+ put_online_mems();
-+ kfree(nodes);
-+ return x + sprintf(buf + x, "\n");
-+}
-+
-+#ifdef CONFIG_SLUB_DEBUG
-+static int any_slab_objects(struct kmem_cache *s)
-+{
-+ int node;
-+ struct kmem_cache_node *n;
-+
-+ for_each_kmem_cache_node(s, node, n)
-+ if (atomic_long_read(&n->total_objects))
-+ return 1;
-+
-+ return 0;
-+}
-+#endif
-+
-+#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
-+#define to_slab(n) container_of(n, struct kmem_cache, kobj)
-+
-+struct slab_attribute {
-+ struct attribute attr;
-+ ssize_t (*show)(struct kmem_cache *s, char *buf);
-+ ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
-+};
-+
-+#define SLAB_ATTR_RO(_name) \
-+ static struct slab_attribute _name##_attr = \
-+ __ATTR(_name, 0400, _name##_show, NULL)
-+
-+#define SLAB_ATTR(_name) \
-+ static struct slab_attribute _name##_attr = \
-+ __ATTR(_name, 0600, _name##_show, _name##_store)
-+
-+static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", s->size);
-+}
-+SLAB_ATTR_RO(slab_size);
-+
-+static ssize_t align_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", s->align);
-+}
-+SLAB_ATTR_RO(align);
-+
-+static ssize_t object_size_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", s->object_size);
-+}
-+SLAB_ATTR_RO(object_size);
-+
-+static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", oo_objects(s->oo));
-+}
-+SLAB_ATTR_RO(objs_per_slab);
-+
-+static ssize_t order_store(struct kmem_cache *s,
-+ const char *buf, size_t length)
-+{
-+ unsigned long order;
-+ int err;
-+
-+ err = kstrtoul(buf, 10, &order);
-+ if (err)
-+ return err;
-+
-+ if (order > slub_max_order || order < slub_min_order)
-+ return -EINVAL;
-+
-+ calculate_sizes(s, order);
-+ return length;
-+}
-+
-+static ssize_t order_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", oo_order(s->oo));
-+}
-+SLAB_ATTR(order);
-+
-+static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%lu\n", s->min_partial);
-+}
-+
-+static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
-+ size_t length)
-+{
-+ unsigned long min;
-+ int err;
-+
-+ err = kstrtoul(buf, 10, &min);
-+ if (err)
-+ return err;
-+
-+ set_min_partial(s, min);
-+ return length;
-+}
-+SLAB_ATTR(min_partial);
-+
-+static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%u\n", s->cpu_partial);
-+}
-+
-+static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
-+ size_t length)
-+{
-+ unsigned long objects;
-+ int err;
-+
-+ err = kstrtoul(buf, 10, &objects);
-+ if (err)
-+ return err;
-+ if (objects && !kmem_cache_has_cpu_partial(s))
-+ return -EINVAL;
-+
-+ s->cpu_partial = objects;
-+ flush_all(s);
-+ return length;
-+}
-+SLAB_ATTR(cpu_partial);
-+
-+static ssize_t ctor_show(struct kmem_cache *s, char *buf)
-+{
-+ if (!s->ctor)
-+ return 0;
-+ return sprintf(buf, "%pS\n", s->ctor);
-+}
-+SLAB_ATTR_RO(ctor);
-+
-+static ssize_t aliases_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
-+}
-+SLAB_ATTR_RO(aliases);
-+
-+static ssize_t partial_show(struct kmem_cache *s, char *buf)
-+{
-+ return show_slab_objects(s, buf, SO_PARTIAL);
-+}
-+SLAB_ATTR_RO(partial);
-+
-+static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
-+{
-+ return show_slab_objects(s, buf, SO_CPU);
-+}
-+SLAB_ATTR_RO(cpu_slabs);
-+
-+static ssize_t objects_show(struct kmem_cache *s, char *buf)
-+{
-+ return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
-+}
-+SLAB_ATTR_RO(objects);
-+
-+static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
-+{
-+ return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
-+}
-+SLAB_ATTR_RO(objects_partial);
-+
-+static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
-+{
-+ int objects = 0;
-+ int pages = 0;
-+ int cpu;
-+ int len;
-+
-+ for_each_online_cpu(cpu) {
-+ struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;
-+
-+ if (page) {
-+ pages += page->pages;
-+ objects += page->pobjects;
-+ }
-+ }
-+
-+ len = sprintf(buf, "%d(%d)", objects, pages);
-+
-+#ifdef CONFIG_SMP
-+ for_each_online_cpu(cpu) {
-+ struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;
-+
-+ if (page && len < PAGE_SIZE - 20)
-+ len += sprintf(buf + len, " C%d=%d(%d)", cpu,
-+ page->pobjects, page->pages);
-+ }
-+#endif
-+ return len + sprintf(buf + len, "\n");
-+}
-+SLAB_ATTR_RO(slabs_cpu_partial);
-+
-+static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
-+}
-+
-+static ssize_t reclaim_account_store(struct kmem_cache *s,
-+ const char *buf, size_t length)
-+{
-+ s->flags &= ~SLAB_RECLAIM_ACCOUNT;
-+ if (buf[0] == '1')
-+ s->flags |= SLAB_RECLAIM_ACCOUNT;
-+ return length;
-+}
-+SLAB_ATTR(reclaim_account);
-+
-+static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
-+}
-+SLAB_ATTR_RO(hwcache_align);
-+
-+#ifdef CONFIG_ZONE_DMA
-+static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
-+}
-+SLAB_ATTR_RO(cache_dma);
-+#endif
-+
-+static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
-+}
-+SLAB_ATTR_RO(destroy_by_rcu);
-+
-+static ssize_t reserved_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", s->reserved);
-+}
-+SLAB_ATTR_RO(reserved);
-+
-+#ifdef CONFIG_SLUB_DEBUG
-+static ssize_t slabs_show(struct kmem_cache *s, char *buf)
-+{
-+ return show_slab_objects(s, buf, SO_ALL);
-+}
-+SLAB_ATTR_RO(slabs);
-+
-+static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
-+{
-+ return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
-+}
-+SLAB_ATTR_RO(total_objects);
-+
-+static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
-+}
-+
-+static ssize_t sanity_checks_store(struct kmem_cache *s,
-+ const char *buf, size_t length)
-+{
-+ s->flags &= ~SLAB_DEBUG_FREE;
-+ if (buf[0] == '1') {
-+ s->flags &= ~__CMPXCHG_DOUBLE;
-+ s->flags |= SLAB_DEBUG_FREE;
-+ }
-+ return length;
-+}
-+SLAB_ATTR(sanity_checks);
-+
-+static ssize_t trace_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
-+}
-+
-+static ssize_t trace_store(struct kmem_cache *s, const char *buf,
-+ size_t length)
-+{
-+ /*
-+ * Tracing a merged cache is going to give confusing results
-+ * as well as cause other issues like converting a mergeable
-+ * cache into an umergeable one.
-+ */
-+ if (s->refcount > 1)
-+ return -EINVAL;
-+
-+ s->flags &= ~SLAB_TRACE;
-+ if (buf[0] == '1') {
-+ s->flags &= ~__CMPXCHG_DOUBLE;
-+ s->flags |= SLAB_TRACE;
-+ }
-+ return length;
-+}
-+SLAB_ATTR(trace);
-+
-+static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
-+}
-+
-+static ssize_t red_zone_store(struct kmem_cache *s,
-+ const char *buf, size_t length)
-+{
-+ if (any_slab_objects(s))
-+ return -EBUSY;
-+
-+ s->flags &= ~SLAB_RED_ZONE;
-+ if (buf[0] == '1') {
-+ s->flags &= ~__CMPXCHG_DOUBLE;
-+ s->flags |= SLAB_RED_ZONE;
-+ }
-+ calculate_sizes(s, -1);
-+ return length;
-+}
-+SLAB_ATTR(red_zone);
-+
-+static ssize_t poison_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
-+}
-+
-+static ssize_t poison_store(struct kmem_cache *s,
-+ const char *buf, size_t length)
-+{
-+ if (any_slab_objects(s))
-+ return -EBUSY;
-+
-+ s->flags &= ~SLAB_POISON;
-+ if (buf[0] == '1') {
-+ s->flags &= ~__CMPXCHG_DOUBLE;
-+ s->flags |= SLAB_POISON;
-+ }
-+ calculate_sizes(s, -1);
-+ return length;
-+}
-+SLAB_ATTR(poison);
-+
-+static ssize_t store_user_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
-+}
-+
-+static ssize_t store_user_store(struct kmem_cache *s,
-+ const char *buf, size_t length)
-+{
-+ if (any_slab_objects(s))
-+ return -EBUSY;
-+
-+ s->flags &= ~SLAB_STORE_USER;
-+ if (buf[0] == '1') {
-+ s->flags &= ~__CMPXCHG_DOUBLE;
-+ s->flags |= SLAB_STORE_USER;
-+ }
-+ calculate_sizes(s, -1);
-+ return length;
-+}
-+SLAB_ATTR(store_user);
-+
-+static ssize_t validate_show(struct kmem_cache *s, char *buf)
-+{
-+ return 0;
-+}
-+
-+static ssize_t validate_store(struct kmem_cache *s,
-+ const char *buf, size_t length)
-+{
-+ int ret = -EINVAL;
-+
-+ if (buf[0] == '1') {
-+ ret = validate_slab_cache(s);
-+ if (ret >= 0)
-+ ret = length;
-+ }
-+ return ret;
-+}
-+SLAB_ATTR(validate);
-+
-+static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
-+{
-+ if (!(s->flags & SLAB_STORE_USER))
-+ return -ENOSYS;
-+ return list_locations(s, buf, TRACK_ALLOC);
-+}
-+SLAB_ATTR_RO(alloc_calls);
-+
-+static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
-+{
-+ if (!(s->flags & SLAB_STORE_USER))
-+ return -ENOSYS;
-+ return list_locations(s, buf, TRACK_FREE);
-+}
-+SLAB_ATTR_RO(free_calls);
-+#endif /* CONFIG_SLUB_DEBUG */
-+
-+#ifdef CONFIG_FAILSLAB
-+static ssize_t failslab_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
-+}
-+
-+static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
-+ size_t length)
-+{
-+ if (s->refcount > 1)
-+ return -EINVAL;
-+
-+ s->flags &= ~SLAB_FAILSLAB;
-+ if (buf[0] == '1')
-+ s->flags |= SLAB_FAILSLAB;
-+ return length;
-+}
-+SLAB_ATTR(failslab);
-+#endif
-+
-+static ssize_t shrink_show(struct kmem_cache *s, char *buf)
-+{
-+ return 0;
-+}
-+
-+static ssize_t shrink_store(struct kmem_cache *s,
-+ const char *buf, size_t length)
-+{
-+ if (buf[0] == '1')
-+ kmem_cache_shrink(s);
-+ else
-+ return -EINVAL;
-+ return length;
-+}
-+SLAB_ATTR(shrink);
-+
-+#ifdef CONFIG_NUMA
-+static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
-+{
-+ return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
-+}
-+
-+static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
-+ const char *buf, size_t length)
-+{
-+ unsigned long ratio;
-+ int err;
-+
-+ err = kstrtoul(buf, 10, &ratio);
-+ if (err)
-+ return err;
-+
-+ if (ratio <= 100)
-+ s->remote_node_defrag_ratio = ratio * 10;
-+
-+ return length;
-+}
-+SLAB_ATTR(remote_node_defrag_ratio);
-+#endif
-+
-+#ifdef CONFIG_SLUB_STATS
-+static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
-+{
-+ unsigned long sum = 0;
-+ int cpu;
-+ int len;
-+ int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);
-+
-+ if (!data)
-+ return -ENOMEM;
-+
-+ for_each_online_cpu(cpu) {
-+ unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
-+
-+ data[cpu] = x;
-+ sum += x;
-+ }
-+
-+ len = sprintf(buf, "%lu", sum);
-+
-+#ifdef CONFIG_SMP
-+ for_each_online_cpu(cpu) {
-+ if (data[cpu] && len < PAGE_SIZE - 20)
-+ len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
-+ }
-+#endif
-+ kfree(data);
-+ return len + sprintf(buf + len, "\n");
-+}
-+
-+static void clear_stat(struct kmem_cache *s, enum stat_item si)
-+{
-+ int cpu;
-+
-+ for_each_online_cpu(cpu)
-+ per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
-+}
-+
-+#define STAT_ATTR(si, text) \
-+static ssize_t text##_show(struct kmem_cache *s, char *buf) \
-+{ \
-+ return show_stat(s, buf, si); \
-+} \
-+static ssize_t text##_store(struct kmem_cache *s, \
-+ const char *buf, size_t length) \
-+{ \
-+ if (buf[0] != '0') \
-+ return -EINVAL; \
-+ clear_stat(s, si); \
-+ return length; \
-+} \
-+SLAB_ATTR(text); \
-+
-+STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
-+STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
-+STAT_ATTR(FREE_FASTPATH, free_fastpath);
-+STAT_ATTR(FREE_SLOWPATH, free_slowpath);
-+STAT_ATTR(FREE_FROZEN, free_frozen);
-+STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
-+STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
-+STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
-+STAT_ATTR(ALLOC_SLAB, alloc_slab);
-+STAT_ATTR(ALLOC_REFILL, alloc_refill);
-+STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
-+STAT_ATTR(FREE_SLAB, free_slab);
-+STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
-+STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
-+STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
-+STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
-+STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
-+STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
-+STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
-+STAT_ATTR(ORDER_FALLBACK, order_fallback);
-+STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
-+STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
-+STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
-+STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
-+STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
-+STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
-+#endif
-+
-+static struct attribute *slab_attrs[] = {
-+ &slab_size_attr.attr,
-+ &object_size_attr.attr,
-+ &objs_per_slab_attr.attr,
-+ &order_attr.attr,
-+ &min_partial_attr.attr,
-+ &cpu_partial_attr.attr,
-+ &objects_attr.attr,
-+ &objects_partial_attr.attr,
-+ &partial_attr.attr,
-+ &cpu_slabs_attr.attr,
-+ &ctor_attr.attr,
-+ &aliases_attr.attr,
-+ &align_attr.attr,
-+ &hwcache_align_attr.attr,
-+ &reclaim_account_attr.attr,
-+ &destroy_by_rcu_attr.attr,
-+ &shrink_attr.attr,
-+ &reserved_attr.attr,
-+ &slabs_cpu_partial_attr.attr,
-+#ifdef CONFIG_SLUB_DEBUG
-+ &total_objects_attr.attr,
-+ &slabs_attr.attr,
-+ &sanity_checks_attr.attr,
-+ &trace_attr.attr,
-+ &red_zone_attr.attr,
-+ &poison_attr.attr,
-+ &store_user_attr.attr,
-+ &validate_attr.attr,
-+ &alloc_calls_attr.attr,
-+ &free_calls_attr.attr,
-+#endif
-+#ifdef CONFIG_ZONE_DMA
-+ &cache_dma_attr.attr,
-+#endif
-+#ifdef CONFIG_NUMA
-+ &remote_node_defrag_ratio_attr.attr,
-+#endif
-+#ifdef CONFIG_SLUB_STATS
-+ &alloc_fastpath_attr.attr,
-+ &alloc_slowpath_attr.attr,
-+ &free_fastpath_attr.attr,
-+ &free_slowpath_attr.attr,
-+ &free_frozen_attr.attr,
-+ &free_add_partial_attr.attr,
-+ &free_remove_partial_attr.attr,
-+ &alloc_from_partial_attr.attr,
-+ &alloc_slab_attr.attr,
-+ &alloc_refill_attr.attr,
-+ &alloc_node_mismatch_attr.attr,
-+ &free_slab_attr.attr,
-+ &cpuslab_flush_attr.attr,
-+ &deactivate_full_attr.attr,
-+ &deactivate_empty_attr.attr,
-+ &deactivate_to_head_attr.attr,
-+ &deactivate_to_tail_attr.attr,
-+ &deactivate_remote_frees_attr.attr,
-+ &deactivate_bypass_attr.attr,
-+ &order_fallback_attr.attr,
-+ &cmpxchg_double_fail_attr.attr,
-+ &cmpxchg_double_cpu_fail_attr.attr,
-+ &cpu_partial_alloc_attr.attr,
-+ &cpu_partial_free_attr.attr,
-+ &cpu_partial_node_attr.attr,
-+ &cpu_partial_drain_attr.attr,
-+#endif
-+#ifdef CONFIG_FAILSLAB
-+ &failslab_attr.attr,
-+#endif
-+
-+ NULL
-+};
-+
-+static struct attribute_group slab_attr_group = {
-+ .attrs = slab_attrs,
-+};
-+
-+static ssize_t slab_attr_show(struct kobject *kobj,
-+ struct attribute *attr,
-+ char *buf)
-+{
-+ struct slab_attribute *attribute;
-+ struct kmem_cache *s;
-+ int err;
-+
-+ attribute = to_slab_attr(attr);
-+ s = to_slab(kobj);
-+
-+ if (!attribute->show)
-+ return -EIO;
-+
-+ err = attribute->show(s, buf);
-+
-+ return err;
-+}
-+
-+static ssize_t slab_attr_store(struct kobject *kobj,
-+ struct attribute *attr,
-+ const char *buf, size_t len)
-+{
-+ struct slab_attribute *attribute;
-+ struct kmem_cache *s;
-+ int err;
-+
-+ attribute = to_slab_attr(attr);
-+ s = to_slab(kobj);
-+
-+ if (!attribute->store)
-+ return -EIO;
-+
-+ err = attribute->store(s, buf, len);
-+#ifdef CONFIG_MEMCG_KMEM
-+ if (slab_state >= FULL && err >= 0 && is_root_cache(s)) {
-+ struct kmem_cache *c;
-+
-+ mutex_lock(&slab_mutex);
-+ if (s->max_attr_size < len)
-+ s->max_attr_size = len;
-+
-+ /*
-+ * This is a best effort propagation, so this function's return
-+ * value will be determined by the parent cache only. This is
-+ * basically because not all attributes will have a well
-+ * defined semantics for rollbacks - most of the actions will
-+ * have permanent effects.
-+ *
-+ * Returning the error value of any of the children that fail
-+ * is not 100 % defined, in the sense that users seeing the
-+ * error code won't be able to know anything about the state of
-+ * the cache.
-+ *
-+ * Only returning the error code for the parent cache at least
-+ * has well defined semantics. The cache being written to
-+ * directly either failed or succeeded, in which case we loop
-+ * through the descendants with best-effort propagation.
-+ */
-+ for_each_memcg_cache(c, s)
-+ attribute->store(c, buf, len);
-+ mutex_unlock(&slab_mutex);
-+ }
-+#endif
-+ return err;
-+}
-+
-+static void memcg_propagate_slab_attrs(struct kmem_cache *s)
-+{
-+#ifdef CONFIG_MEMCG_KMEM
-+ int i;
-+ char *buffer = NULL;
-+ struct kmem_cache *root_cache;
-+
-+ if (is_root_cache(s))
-+ return;
-+
-+ root_cache = s->memcg_params.root_cache;
-+
-+ /*
-+ * This mean this cache had no attribute written. Therefore, no point
-+ * in copying default values around
-+ */
-+ if (!root_cache->max_attr_size)
-+ return;
-+
-+ for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) {
-+ char mbuf[64];
-+ char *buf;
-+ struct slab_attribute *attr = to_slab_attr(slab_attrs[i]);
-+
-+ if (!attr || !attr->store || !attr->show)
-+ continue;
-+
-+ /*
-+ * It is really bad that we have to allocate here, so we will
-+ * do it only as a fallback. If we actually allocate, though,
-+ * we can just use the allocated buffer until the end.
-+ *
-+ * Most of the slub attributes will tend to be very small in
-+ * size, but sysfs allows buffers up to a page, so they can
-+ * theoretically happen.
-+ */
-+ if (buffer)
-+ buf = buffer;
-+ else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf))
-+ buf = mbuf;
-+ else {
-+ buffer = (char *) get_zeroed_page(GFP_KERNEL);
-+ if (WARN_ON(!buffer))
-+ continue;
-+ buf = buffer;
-+ }
-+
-+ attr->show(root_cache, buf);
-+ attr->store(s, buf, strlen(buf));
-+ }
-+
-+ if (buffer)
-+ free_page((unsigned long)buffer);
-+#endif
-+}
-+
-+static void kmem_cache_release(struct kobject *k)
-+{
-+ slab_kmem_cache_release(to_slab(k));
-+}
-+
-+static const struct sysfs_ops slab_sysfs_ops = {
-+ .show = slab_attr_show,
-+ .store = slab_attr_store,
-+};
-+
-+static struct kobj_type slab_ktype = {
-+ .sysfs_ops = &slab_sysfs_ops,
-+ .release = kmem_cache_release,
-+};
-+
-+static int uevent_filter(struct kset *kset, struct kobject *kobj)
-+{
-+ struct kobj_type *ktype = get_ktype(kobj);
-+
-+ if (ktype == &slab_ktype)
-+ return 1;
-+ return 0;
-+}
-+
-+static const struct kset_uevent_ops slab_uevent_ops = {
-+ .filter = uevent_filter,
-+};
-+
-+static struct kset *slab_kset;
-+
-+static inline struct kset *cache_kset(struct kmem_cache *s)
-+{
-+#ifdef CONFIG_MEMCG_KMEM
-+ if (!is_root_cache(s))
-+ return s->memcg_params.root_cache->memcg_kset;
-+#endif
-+ return slab_kset;
-+}
-+
-+#define ID_STR_LENGTH 64
-+
-+/* Create a unique string id for a slab cache:
-+ *
-+ * Format :[flags-]size
-+ */
-+static char *create_unique_id(struct kmem_cache *s)
-+{
-+ char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
-+ char *p = name;
-+
-+ BUG_ON(!name);
-+
-+ *p++ = ':';
-+ /*
-+ * First flags affecting slabcache operations. We will only
-+ * get here for aliasable slabs so we do not need to support
-+ * too many flags. The flags here must cover all flags that
-+ * are matched during merging to guarantee that the id is
-+ * unique.
-+ */
-+ if (s->flags & SLAB_CACHE_DMA)
-+ *p++ = 'd';
-+ if (s->flags & SLAB_RECLAIM_ACCOUNT)
-+ *p++ = 'a';
-+ if (s->flags & SLAB_DEBUG_FREE)
-+ *p++ = 'F';
-+ if (!(s->flags & SLAB_NOTRACK))
-+ *p++ = 't';
-+ if (p != name + 1)
-+ *p++ = '-';
-+ p += sprintf(p, "%07d", s->size);
-+
-+ BUG_ON(p > name + ID_STR_LENGTH - 1);
-+ return name;
-+}
-+
-+static int sysfs_slab_add(struct kmem_cache *s)
-+{
-+ int err;
-+ const char *name;
-+ int unmergeable = slab_unmergeable(s);
-+
-+ if (unmergeable) {
-+ /*
-+ * Slabcache can never be merged so we can use the name proper.
-+ * This is typically the case for debug situations. In that
-+ * case we can catch duplicate names easily.
-+ */
-+ sysfs_remove_link(&slab_kset->kobj, s->name);
-+ name = s->name;
-+ } else {
-+ /*
-+ * Create a unique name for the slab as a target
-+ * for the symlinks.
-+ */
-+ name = create_unique_id(s);
-+ }
-+
-+ s->kobj.kset = cache_kset(s);
-+ err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
-+ if (err)
-+ goto out_put_kobj;
-+
-+ err = sysfs_create_group(&s->kobj, &slab_attr_group);
-+ if (err)
-+ goto out_del_kobj;
-+
-+#ifdef CONFIG_MEMCG_KMEM
-+ if (is_root_cache(s)) {
-+ s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
-+ if (!s->memcg_kset) {
-+ err = -ENOMEM;
-+ goto out_del_kobj;
-+ }
-+ }
-+#endif
-+
-+ kobject_uevent(&s->kobj, KOBJ_ADD);
-+ if (!unmergeable) {
-+ /* Setup first alias */
-+ sysfs_slab_alias(s, s->name);
-+ }
-+out:
-+ if (!unmergeable)
-+ kfree(name);
-+ return err;
-+out_del_kobj:
-+ kobject_del(&s->kobj);
-+out_put_kobj:
-+ kobject_put(&s->kobj);
-+ goto out;
-+}
-+
-+void sysfs_slab_remove(struct kmem_cache *s)
-+{
-+ if (slab_state < FULL)
-+ /*
-+ * Sysfs has not been setup yet so no need to remove the
-+ * cache from sysfs.
-+ */
-+ return;
-+
-+#ifdef CONFIG_MEMCG_KMEM
-+ kset_unregister(s->memcg_kset);
-+#endif
-+ kobject_uevent(&s->kobj, KOBJ_REMOVE);
-+ kobject_del(&s->kobj);
-+ kobject_put(&s->kobj);
-+}
-+
-+/*
-+ * Need to buffer aliases during bootup until sysfs becomes
-+ * available lest we lose that information.
-+ */
-+struct saved_alias {
-+ struct kmem_cache *s;
-+ const char *name;
-+ struct saved_alias *next;
-+};
-+
-+static struct saved_alias *alias_list;
-+
-+static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
-+{
-+ struct saved_alias *al;
-+
-+ if (slab_state == FULL) {
-+ /*
-+ * If we have a leftover link then remove it.
-+ */
-+ sysfs_remove_link(&slab_kset->kobj, name);
-+ return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
-+ }
-+
-+ al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
-+ if (!al)
-+ return -ENOMEM;
-+
-+ al->s = s;
-+ al->name = name;
-+ al->next = alias_list;
-+ alias_list = al;
-+ return 0;
-+}
-+
-+static int __init slab_sysfs_init(void)
-+{
-+ struct kmem_cache *s;
-+ int err;
-+
-+ mutex_lock(&slab_mutex);
-+
-+ slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
-+ if (!slab_kset) {
-+ mutex_unlock(&slab_mutex);
-+ pr_err("Cannot register slab subsystem.\n");
-+ return -ENOSYS;
-+ }
-+
-+ slab_state = FULL;
-+
-+ list_for_each_entry(s, &slab_caches, list) {
-+ err = sysfs_slab_add(s);
-+ if (err)
-+ pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
-+ s->name);
-+ }
-+
-+ while (alias_list) {
-+ struct saved_alias *al = alias_list;
-+
-+ alias_list = alias_list->next;
-+ err = sysfs_slab_alias(al->s, al->name);
-+ if (err)
-+ pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
-+ al->name);
-+ kfree(al);
-+ }
-+
-+ mutex_unlock(&slab_mutex);
-+ resiliency_test();
-+ return 0;
-+}
-+
-+__initcall(slab_sysfs_init);
-+#endif /* CONFIG_SYSFS */
-+
-+/*
-+ * The /proc/slabinfo ABI
-+ */
-+#ifdef CONFIG_SLABINFO
-+void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
-+{
-+ unsigned long nr_slabs = 0;
-+ unsigned long nr_objs = 0;
-+ unsigned long nr_free = 0;
-+ int node;
-+ struct kmem_cache_node *n;
-+
-+ for_each_kmem_cache_node(s, node, n) {
-+ nr_slabs += node_nr_slabs(n);
-+ nr_objs += node_nr_objs(n);
-+ nr_free += count_partial(n, count_free);
-+ }
-+
-+ sinfo->active_objs = nr_objs - nr_free;
-+ sinfo->num_objs = nr_objs;
-+ sinfo->active_slabs = nr_slabs;
-+ sinfo->num_slabs = nr_slabs;
-+ sinfo->objects_per_slab = oo_objects(s->oo);
-+ sinfo->cache_order = oo_order(s->oo);
-+}
-+
-+void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
-+{
-+}
-+
-+ssize_t slabinfo_write(struct file *file, const char __user *buffer,
-+ size_t count, loff_t *ppos)
-+{
-+ return -EIO;
-+}
-+#endif /* CONFIG_SLABINFO */
diff -Nur linux-4.1.10.orig/mm/swap.c linux-4.1.10/mm/swap.c
--- linux-4.1.10.orig/mm/swap.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/swap.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/swap.c 2015-10-12 22:33:32.336672790 +0200
@@ -32,6 +32,7 @@
#include <linux/gfp.h>
#include <linux/uio.h>
@@ -62047,7 +26260,7 @@ diff -Nur linux-4.1.10.orig/mm/swap.c linux-4.1.10/mm/swap.c
static void lru_add_drain_per_cpu(struct work_struct *dummy)
diff -Nur linux-4.1.10.orig/mm/truncate.c linux-4.1.10/mm/truncate.c
--- linux-4.1.10.orig/mm/truncate.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/truncate.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/truncate.c 2015-10-12 22:33:32.336672790 +0200
@@ -56,8 +56,11 @@
* protected by mapping->tree_lock.
*/
@@ -62064,7 +26277,7 @@ diff -Nur linux-4.1.10.orig/mm/truncate.c linux-4.1.10/mm/truncate.c
spin_unlock_irq(&mapping->tree_lock);
diff -Nur linux-4.1.10.orig/mm/vmalloc.c linux-4.1.10/mm/vmalloc.c
--- linux-4.1.10.orig/mm/vmalloc.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/vmalloc.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/vmalloc.c 2015-10-12 22:33:32.336672790 +0200
@@ -819,7 +819,7 @@
struct vmap_block *vb;
struct vmap_area *va;
@@ -62118,7 +26331,7 @@ diff -Nur linux-4.1.10.orig/mm/vmalloc.c linux-4.1.10/mm/vmalloc.c
/* Allocate new block if nothing was found */
diff -Nur linux-4.1.10.orig/mm/vmstat.c linux-4.1.10/mm/vmstat.c
--- linux-4.1.10.orig/mm/vmstat.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/vmstat.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/vmstat.c 2015-10-12 22:33:32.336672790 +0200
@@ -226,6 +226,7 @@
long x;
long t;
@@ -62169,7 +26382,7 @@ diff -Nur linux-4.1.10.orig/mm/vmstat.c linux-4.1.10/mm/vmstat.c
void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
diff -Nur linux-4.1.10.orig/mm/workingset.c linux-4.1.10/mm/workingset.c
--- linux-4.1.10.orig/mm/workingset.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/mm/workingset.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/mm/workingset.c 2015-10-12 22:33:32.336672790 +0200
@@ -264,7 +264,8 @@
* point where they would still be useful.
*/
@@ -62239,7 +26452,7 @@ diff -Nur linux-4.1.10.orig/mm/workingset.c linux-4.1.10/mm/workingset.c
}
diff -Nur linux-4.1.10.orig/net/core/dev.c linux-4.1.10/net/core/dev.c
--- linux-4.1.10.orig/net/core/dev.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/core/dev.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/core/dev.c 2015-10-12 22:33:32.340672525 +0200
@@ -184,6 +184,7 @@
static DEFINE_HASHTABLE(napi_hash, 8);
@@ -62526,7535 +26739,9 @@ diff -Nur linux-4.1.10.orig/net/core/dev.c linux-4.1.10/net/core/dev.c
INIT_LIST_HEAD(&sd->poll_list);
sd->output_queue_tailp = &sd->output_queue;
#ifdef CONFIG_RPS
-diff -Nur linux-4.1.10.orig/net/core/dev.c.orig linux-4.1.10/net/core/dev.c.orig
---- linux-4.1.10.orig/net/core/dev.c.orig 1970-01-01 01:00:00.000000000 +0100
-+++ linux-4.1.10/net/core/dev.c.orig 2015-10-03 13:49:38.000000000 +0200
-@@ -0,0 +1,7522 @@
-+/*
-+ * NET3 Protocol independent device support routines.
-+ *
-+ * This program is free software; you can redistribute it and/or
-+ * modify it under the terms of the GNU General Public License
-+ * as published by the Free Software Foundation; either version
-+ * 2 of the License, or (at your option) any later version.
-+ *
-+ * Derived from the non IP parts of dev.c 1.0.19
-+ * Authors: Ross Biro
-+ * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
-+ * Mark Evans, <evansmp@uhura.aston.ac.uk>
-+ *
-+ * Additional Authors:
-+ * Florian la Roche <rzsfl@rz.uni-sb.de>
-+ * Alan Cox <gw4pts@gw4pts.ampr.org>
-+ * David Hinds <dahinds@users.sourceforge.net>
-+ * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
-+ * Adam Sulmicki <adam@cfar.umd.edu>
-+ * Pekka Riikonen <priikone@poesidon.pspt.fi>
-+ *
-+ * Changes:
-+ * D.J. Barrow : Fixed bug where dev->refcnt gets set
-+ * to 2 if register_netdev gets called
-+ * before net_dev_init & also removed a
-+ * few lines of code in the process.
-+ * Alan Cox : device private ioctl copies fields back.
-+ * Alan Cox : Transmit queue code does relevant
-+ * stunts to keep the queue safe.
-+ * Alan Cox : Fixed double lock.
-+ * Alan Cox : Fixed promisc NULL pointer trap
-+ * ???????? : Support the full private ioctl range
-+ * Alan Cox : Moved ioctl permission check into
-+ * drivers
-+ * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
-+ * Alan Cox : 100 backlog just doesn't cut it when
-+ * you start doing multicast video 8)
-+ * Alan Cox : Rewrote net_bh and list manager.
-+ * Alan Cox : Fix ETH_P_ALL echoback lengths.
-+ * Alan Cox : Took out transmit every packet pass
-+ * Saved a few bytes in the ioctl handler
-+ * Alan Cox : Network driver sets packet type before
-+ * calling netif_rx. Saves a function
-+ * call a packet.
-+ * Alan Cox : Hashed net_bh()
-+ * Richard Kooijman: Timestamp fixes.
-+ * Alan Cox : Wrong field in SIOCGIFDSTADDR
-+ * Alan Cox : Device lock protection.
-+ * Alan Cox : Fixed nasty side effect of device close
-+ * changes.
-+ * Rudi Cilibrasi : Pass the right thing to
-+ * set_mac_address()
-+ * Dave Miller : 32bit quantity for the device lock to
-+ * make it work out on a Sparc.
-+ * Bjorn Ekwall : Added KERNELD hack.
-+ * Alan Cox : Cleaned up the backlog initialise.
-+ * Craig Metz : SIOCGIFCONF fix if space for under
-+ * 1 device.
-+ * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
-+ * is no device open function.
-+ * Andi Kleen : Fix error reporting for SIOCGIFCONF
-+ * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
-+ * Cyrus Durgin : Cleaned for KMOD
-+ * Adam Sulmicki : Bug Fix : Network Device Unload
-+ * A network device unload needs to purge
-+ * the backlog queue.
-+ * Paul Rusty Russell : SIOCSIFNAME
-+ * Pekka Riikonen : Netdev boot-time settings code
-+ * Andrew Morton : Make unregister_netdevice wait
-+ * indefinitely on dev->refcnt
-+ * J Hadi Salim : - Backlog queue sampling
-+ * - netif_rx() feedback
-+ */
-+
-+#include <asm/uaccess.h>
-+#include <linux/bitops.h>
-+#include <linux/capability.h>
-+#include <linux/cpu.h>
-+#include <linux/types.h>
-+#include <linux/kernel.h>
-+#include <linux/hash.h>
-+#include <linux/slab.h>
-+#include <linux/sched.h>
-+#include <linux/mutex.h>
-+#include <linux/string.h>
-+#include <linux/mm.h>
-+#include <linux/socket.h>
-+#include <linux/sockios.h>
-+#include <linux/errno.h>
-+#include <linux/interrupt.h>
-+#include <linux/if_ether.h>
-+#include <linux/netdevice.h>
-+#include <linux/etherdevice.h>
-+#include <linux/ethtool.h>
-+#include <linux/notifier.h>
-+#include <linux/skbuff.h>
-+#include <net/net_namespace.h>
-+#include <net/sock.h>
-+#include <linux/rtnetlink.h>
-+#include <linux/stat.h>
-+#include <net/dst.h>
-+#include <net/pkt_sched.h>
-+#include <net/checksum.h>
-+#include <net/xfrm.h>
-+#include <linux/highmem.h>
-+#include <linux/init.h>
-+#include <linux/module.h>
-+#include <linux/netpoll.h>
-+#include <linux/rcupdate.h>
-+#include <linux/delay.h>
-+#include <net/iw_handler.h>
-+#include <asm/current.h>
-+#include <linux/audit.h>
-+#include <linux/dmaengine.h>
-+#include <linux/err.h>
-+#include <linux/ctype.h>
-+#include <linux/if_arp.h>
-+#include <linux/if_vlan.h>
-+#include <linux/ip.h>
-+#include <net/ip.h>
-+#include <net/mpls.h>
-+#include <linux/ipv6.h>
-+#include <linux/in.h>
-+#include <linux/jhash.h>
-+#include <linux/random.h>
-+#include <trace/events/napi.h>
-+#include <trace/events/net.h>
-+#include <trace/events/skb.h>
-+#include <linux/pci.h>
-+#include <linux/inetdevice.h>
-+#include <linux/cpu_rmap.h>
-+#include <linux/static_key.h>
-+#include <linux/hashtable.h>
-+#include <linux/vmalloc.h>
-+#include <linux/if_macvlan.h>
-+#include <linux/errqueue.h>
-+#include <linux/hrtimer.h>
-+
-+#include "net-sysfs.h"
-+
-+/* Instead of increasing this, you should create a hash table. */
-+#define MAX_GRO_SKBS 8
-+
-+/* This should be increased if a protocol with a bigger head is added. */
-+#define GRO_MAX_HEAD (MAX_HEADER + 128)
-+
-+static DEFINE_SPINLOCK(ptype_lock);
-+static DEFINE_SPINLOCK(offload_lock);
-+struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
-+struct list_head ptype_all __read_mostly; /* Taps */
-+static struct list_head offload_base __read_mostly;
-+
-+static int netif_rx_internal(struct sk_buff *skb);
-+static int call_netdevice_notifiers_info(unsigned long val,
-+ struct net_device *dev,
-+ struct netdev_notifier_info *info);
-+
-+/*
-+ * The @dev_base_head list is protected by @dev_base_lock and the rtnl
-+ * semaphore.
-+ *
-+ * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
-+ *
-+ * Writers must hold the rtnl semaphore while they loop through the
-+ * dev_base_head list, and hold dev_base_lock for writing when they do the
-+ * actual updates. This allows pure readers to access the list even
-+ * while a writer is preparing to update it.
-+ *
-+ * To put it another way, dev_base_lock is held for writing only to
-+ * protect against pure readers; the rtnl semaphore provides the
-+ * protection against other writers.
-+ *
-+ * See, for example usages, register_netdevice() and
-+ * unregister_netdevice(), which must be called with the rtnl
-+ * semaphore held.
-+ */
-+DEFINE_RWLOCK(dev_base_lock);
-+EXPORT_SYMBOL(dev_base_lock);
-+
-+/* protects napi_hash addition/deletion and napi_gen_id */
-+static DEFINE_SPINLOCK(napi_hash_lock);
-+
-+static unsigned int napi_gen_id;
-+static DEFINE_HASHTABLE(napi_hash, 8);
-+
-+static seqcount_t devnet_rename_seq;
-+
-+static inline void dev_base_seq_inc(struct net *net)
-+{
-+ while (++net->dev_base_seq == 0);
-+}
-+
-+static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
-+{
-+ unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
-+
-+ return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
-+}
-+
-+static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
-+{
-+ return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
-+}
-+
-+static inline void rps_lock(struct softnet_data *sd)
-+{
-+#ifdef CONFIG_RPS
-+ spin_lock(&sd->input_pkt_queue.lock);
-+#endif
-+}
-+
-+static inline void rps_unlock(struct softnet_data *sd)
-+{
-+#ifdef CONFIG_RPS
-+ spin_unlock(&sd->input_pkt_queue.lock);
-+#endif
-+}
-+
-+/* Device list insertion */
-+static void list_netdevice(struct net_device *dev)
-+{
-+ struct net *net = dev_net(dev);
-+
-+ ASSERT_RTNL();
-+
-+ write_lock_bh(&dev_base_lock);
-+ list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
-+ hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
-+ hlist_add_head_rcu(&dev->index_hlist,
-+ dev_index_hash(net, dev->ifindex));
-+ write_unlock_bh(&dev_base_lock);
-+
-+ dev_base_seq_inc(net);
-+}
-+
-+/* Device list removal
-+ * caller must respect a RCU grace period before freeing/reusing dev
-+ */
-+static void unlist_netdevice(struct net_device *dev)
-+{
-+ ASSERT_RTNL();
-+
-+ /* Unlink dev from the device chain */
-+ write_lock_bh(&dev_base_lock);
-+ list_del_rcu(&dev->dev_list);
-+ hlist_del_rcu(&dev->name_hlist);
-+ hlist_del_rcu(&dev->index_hlist);
-+ write_unlock_bh(&dev_base_lock);
-+
-+ dev_base_seq_inc(dev_net(dev));
-+}
-+
-+/*
-+ * Our notifier list
-+ */
-+
-+static RAW_NOTIFIER_HEAD(netdev_chain);
-+
-+/*
-+ * Device drivers call our routines to queue packets here. We empty the
-+ * queue in the local softnet handler.
-+ */
-+
-+DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
-+EXPORT_PER_CPU_SYMBOL(softnet_data);
-+
-+#ifdef CONFIG_LOCKDEP
-+/*
-+ * register_netdevice() inits txq->_xmit_lock and sets lockdep class
-+ * according to dev->type
-+ */
-+static const unsigned short netdev_lock_type[] =
-+ {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
-+ ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
-+ ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
-+ ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
-+ ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
-+ ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
-+ ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
-+ ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
-+ ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
-+ ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
-+ ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
-+ ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
-+ ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
-+ ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
-+ ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
-+
-+static const char *const netdev_lock_name[] =
-+ {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
-+ "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
-+ "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
-+ "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
-+ "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
-+ "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
-+ "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
-+ "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
-+ "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
-+ "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
-+ "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
-+ "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
-+ "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
-+ "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
-+ "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
-+
-+static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
-+static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
-+
-+static inline unsigned short netdev_lock_pos(unsigned short dev_type)
-+{
-+ int i;
-+
-+ for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
-+ if (netdev_lock_type[i] == dev_type)
-+ return i;
-+ /* the last key is used by default */
-+ return ARRAY_SIZE(netdev_lock_type) - 1;
-+}
-+
-+static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
-+ unsigned short dev_type)
-+{
-+ int i;
-+
-+ i = netdev_lock_pos(dev_type);
-+ lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
-+ netdev_lock_name[i]);
-+}
-+
-+static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
-+{
-+ int i;
-+
-+ i = netdev_lock_pos(dev->type);
-+ lockdep_set_class_and_name(&dev->addr_list_lock,
-+ &netdev_addr_lock_key[i],
-+ netdev_lock_name[i]);
-+}
-+#else
-+static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
-+ unsigned short dev_type)
-+{
-+}
-+static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
-+{
-+}
-+#endif
-+
-+/*******************************************************************************
-+
-+ Protocol management and registration routines
-+
-+*******************************************************************************/
-+
-+/*
-+ * Add a protocol ID to the list. Now that the input handler is
-+ * smarter we can dispense with all the messy stuff that used to be
-+ * here.
-+ *
-+ * BEWARE!!! Protocol handlers, mangling input packets,
-+ * MUST BE last in hash buckets and checking protocol handlers
-+ * MUST start from promiscuous ptype_all chain in net_bh.
-+ * It is true now, do not change it.
-+ * Explanation follows: if protocol handler, mangling packet, will
-+ * be the first on list, it is not able to sense, that packet
-+ * is cloned and should be copied-on-write, so that it will
-+ * change it and subsequent readers will get broken packet.
-+ * --ANK (980803)
-+ */
-+
-+static inline struct list_head *ptype_head(const struct packet_type *pt)
-+{
-+ if (pt->type == htons(ETH_P_ALL))
-+ return pt->dev ? &pt->dev->ptype_all : &ptype_all;
-+ else
-+ return pt->dev ? &pt->dev->ptype_specific :
-+ &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
-+}
-+
-+/**
-+ * dev_add_pack - add packet handler
-+ * @pt: packet type declaration
-+ *
-+ * Add a protocol handler to the networking stack. The passed &packet_type
-+ * is linked into kernel lists and may not be freed until it has been
-+ * removed from the kernel lists.
-+ *
-+ * This call does not sleep therefore it can not
-+ * guarantee all CPU's that are in middle of receiving packets
-+ * will see the new packet type (until the next received packet).
-+ */
-+
-+void dev_add_pack(struct packet_type *pt)
-+{
-+ struct list_head *head = ptype_head(pt);
-+
-+ spin_lock(&ptype_lock);
-+ list_add_rcu(&pt->list, head);
-+ spin_unlock(&ptype_lock);
-+}
-+EXPORT_SYMBOL(dev_add_pack);
-+
-+/**
-+ * __dev_remove_pack - remove packet handler
-+ * @pt: packet type declaration
-+ *
-+ * Remove a protocol handler that was previously added to the kernel
-+ * protocol handlers by dev_add_pack(). The passed &packet_type is removed
-+ * from the kernel lists and can be freed or reused once this function
-+ * returns.
-+ *
-+ * The packet type might still be in use by receivers
-+ * and must not be freed until after all the CPU's have gone
-+ * through a quiescent state.
-+ */
-+void __dev_remove_pack(struct packet_type *pt)
-+{
-+ struct list_head *head = ptype_head(pt);
-+ struct packet_type *pt1;
-+
-+ spin_lock(&ptype_lock);
-+
-+ list_for_each_entry(pt1, head, list) {
-+ if (pt == pt1) {
-+ list_del_rcu(&pt->list);
-+ goto out;
-+ }
-+ }
-+
-+ pr_warn("dev_remove_pack: %p not found\n", pt);
-+out:
-+ spin_unlock(&ptype_lock);
-+}
-+EXPORT_SYMBOL(__dev_remove_pack);
-+
-+/**
-+ * dev_remove_pack - remove packet handler
-+ * @pt: packet type declaration
-+ *
-+ * Remove a protocol handler that was previously added to the kernel
-+ * protocol handlers by dev_add_pack(). The passed &packet_type is removed
-+ * from the kernel lists and can be freed or reused once this function
-+ * returns.
-+ *
-+ * This call sleeps to guarantee that no CPU is looking at the packet
-+ * type after return.
-+ */
-+void dev_remove_pack(struct packet_type *pt)
-+{
-+ __dev_remove_pack(pt);
-+
-+ synchronize_net();
-+}
-+EXPORT_SYMBOL(dev_remove_pack);
-+
-+
-+/**
-+ * dev_add_offload - register offload handlers
-+ * @po: protocol offload declaration
-+ *
-+ * Add protocol offload handlers to the networking stack. The passed
-+ * &proto_offload is linked into kernel lists and may not be freed until
-+ * it has been removed from the kernel lists.
-+ *
-+ * This call does not sleep therefore it can not
-+ * guarantee all CPU's that are in middle of receiving packets
-+ * will see the new offload handlers (until the next received packet).
-+ */
-+void dev_add_offload(struct packet_offload *po)
-+{
-+ struct list_head *head = &offload_base;
-+
-+ spin_lock(&offload_lock);
-+ list_add_rcu(&po->list, head);
-+ spin_unlock(&offload_lock);
-+}
-+EXPORT_SYMBOL(dev_add_offload);
-+
-+/**
-+ * __dev_remove_offload - remove offload handler
-+ * @po: packet offload declaration
-+ *
-+ * Remove a protocol offload handler that was previously added to the
-+ * kernel offload handlers by dev_add_offload(). The passed &offload_type
-+ * is removed from the kernel lists and can be freed or reused once this
-+ * function returns.
-+ *
-+ * The packet type might still be in use by receivers
-+ * and must not be freed until after all the CPU's have gone
-+ * through a quiescent state.
-+ */
-+static void __dev_remove_offload(struct packet_offload *po)
-+{
-+ struct list_head *head = &offload_base;
-+ struct packet_offload *po1;
-+
-+ spin_lock(&offload_lock);
-+
-+ list_for_each_entry(po1, head, list) {
-+ if (po == po1) {
-+ list_del_rcu(&po->list);
-+ goto out;
-+ }
-+ }
-+
-+ pr_warn("dev_remove_offload: %p not found\n", po);
-+out:
-+ spin_unlock(&offload_lock);
-+}
-+
-+/**
-+ * dev_remove_offload - remove packet offload handler
-+ * @po: packet offload declaration
-+ *
-+ * Remove a packet offload handler that was previously added to the kernel
-+ * offload handlers by dev_add_offload(). The passed &offload_type is
-+ * removed from the kernel lists and can be freed or reused once this
-+ * function returns.
-+ *
-+ * This call sleeps to guarantee that no CPU is looking at the packet
-+ * type after return.
-+ */
-+void dev_remove_offload(struct packet_offload *po)
-+{
-+ __dev_remove_offload(po);
-+
-+ synchronize_net();
-+}
-+EXPORT_SYMBOL(dev_remove_offload);
-+
-+/******************************************************************************
-+
-+ Device Boot-time Settings Routines
-+
-+*******************************************************************************/
-+
-+/* Boot time configuration table */
-+static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
-+
-+/**
-+ * netdev_boot_setup_add - add new setup entry
-+ * @name: name of the device
-+ * @map: configured settings for the device
-+ *
-+ * Adds new setup entry to the dev_boot_setup list. The function
-+ * returns 0 on error and 1 on success. This is a generic routine to
-+ * all netdevices.
-+ */
-+static int netdev_boot_setup_add(char *name, struct ifmap *map)
-+{
-+ struct netdev_boot_setup *s;
-+ int i;
-+
-+ s = dev_boot_setup;
-+ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
-+ if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
-+ memset(s[i].name, 0, sizeof(s[i].name));
-+ strlcpy(s[i].name, name, IFNAMSIZ);
-+ memcpy(&s[i].map, map, sizeof(s[i].map));
-+ break;
-+ }
-+ }
-+
-+ return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
-+}
-+
-+/**
-+ * netdev_boot_setup_check - check boot time settings
-+ * @dev: the netdevice
-+ *
-+ * Check boot time settings for the device.
-+ * The found settings are set for the device to be used
-+ * later in the device probing.
-+ * Returns 0 if no settings found, 1 if they are.
-+ */
-+int netdev_boot_setup_check(struct net_device *dev)
-+{
-+ struct netdev_boot_setup *s = dev_boot_setup;
-+ int i;
-+
-+ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
-+ if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
-+ !strcmp(dev->name, s[i].name)) {
-+ dev->irq = s[i].map.irq;
-+ dev->base_addr = s[i].map.base_addr;
-+ dev->mem_start = s[i].map.mem_start;
-+ dev->mem_end = s[i].map.mem_end;
-+ return 1;
-+ }
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(netdev_boot_setup_check);
-+
-+
-+/**
-+ * netdev_boot_base - get address from boot time settings
-+ * @prefix: prefix for network device
-+ * @unit: id for network device
-+ *
-+ * Check boot time settings for the base address of device.
-+ * The found settings are set for the device to be used
-+ * later in the device probing.
-+ * Returns 0 if no settings found.
-+ */
-+unsigned long netdev_boot_base(const char *prefix, int unit)
-+{
-+ const struct netdev_boot_setup *s = dev_boot_setup;
-+ char name[IFNAMSIZ];
-+ int i;
-+
-+ sprintf(name, "%s%d", prefix, unit);
-+
-+ /*
-+ * If device already registered then return base of 1
-+ * to indicate not to probe for this interface
-+ */
-+ if (__dev_get_by_name(&init_net, name))
-+ return 1;
-+
-+ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
-+ if (!strcmp(name, s[i].name))
-+ return s[i].map.base_addr;
-+ return 0;
-+}
-+
-+/*
-+ * Saves at boot time configured settings for any netdevice.
-+ */
-+int __init netdev_boot_setup(char *str)
-+{
-+ int ints[5];
-+ struct ifmap map;
-+
-+ str = get_options(str, ARRAY_SIZE(ints), ints);
-+ if (!str || !*str)
-+ return 0;
-+
-+ /* Save settings */
-+ memset(&map, 0, sizeof(map));
-+ if (ints[0] > 0)
-+ map.irq = ints[1];
-+ if (ints[0] > 1)
-+ map.base_addr = ints[2];
-+ if (ints[0] > 2)
-+ map.mem_start = ints[3];
-+ if (ints[0] > 3)
-+ map.mem_end = ints[4];
-+
-+ /* Add new entry to the list */
-+ return netdev_boot_setup_add(str, &map);
-+}
-+
-+__setup("netdev=", netdev_boot_setup);
-+
-+/*******************************************************************************
-+
-+ Device Interface Subroutines
-+
-+*******************************************************************************/
-+
-+/**
-+ * dev_get_iflink - get 'iflink' value of a interface
-+ * @dev: targeted interface
-+ *
-+ * Indicates the ifindex the interface is linked to.
-+ * Physical interfaces have the same 'ifindex' and 'iflink' values.
-+ */
-+
-+int dev_get_iflink(const struct net_device *dev)
-+{
-+ if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
-+ return dev->netdev_ops->ndo_get_iflink(dev);
-+
-+ return dev->ifindex;
-+}
-+EXPORT_SYMBOL(dev_get_iflink);
-+
-+/**
-+ * __dev_get_by_name - find a device by its name
-+ * @net: the applicable net namespace
-+ * @name: name to find
-+ *
-+ * Find an interface by name. Must be called under RTNL semaphore
-+ * or @dev_base_lock. If the name is found a pointer to the device
-+ * is returned. If the name is not found then %NULL is returned. The
-+ * reference counters are not incremented so the caller must be
-+ * careful with locks.
-+ */
-+
-+struct net_device *__dev_get_by_name(struct net *net, const char *name)
-+{
-+ struct net_device *dev;
-+ struct hlist_head *head = dev_name_hash(net, name);
-+
-+ hlist_for_each_entry(dev, head, name_hlist)
-+ if (!strncmp(dev->name, name, IFNAMSIZ))
-+ return dev;
-+
-+ return NULL;
-+}
-+EXPORT_SYMBOL(__dev_get_by_name);
-+
-+/**
-+ * dev_get_by_name_rcu - find a device by its name
-+ * @net: the applicable net namespace
-+ * @name: name to find
-+ *
-+ * Find an interface by name.
-+ * If the name is found a pointer to the device is returned.
-+ * If the name is not found then %NULL is returned.
-+ * The reference counters are not incremented so the caller must be
-+ * careful with locks. The caller must hold RCU lock.
-+ */
-+
-+struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
-+{
-+ struct net_device *dev;
-+ struct hlist_head *head = dev_name_hash(net, name);
-+
-+ hlist_for_each_entry_rcu(dev, head, name_hlist)
-+ if (!strncmp(dev->name, name, IFNAMSIZ))
-+ return dev;
-+
-+ return NULL;
-+}
-+EXPORT_SYMBOL(dev_get_by_name_rcu);
-+
-+/**
-+ * dev_get_by_name - find a device by its name
-+ * @net: the applicable net namespace
-+ * @name: name to find
-+ *
-+ * Find an interface by name. This can be called from any
-+ * context and does its own locking. The returned handle has
-+ * the usage count incremented and the caller must use dev_put() to
-+ * release it when it is no longer needed. %NULL is returned if no
-+ * matching device is found.
-+ */
-+
-+struct net_device *dev_get_by_name(struct net *net, const char *name)
-+{
-+ struct net_device *dev;
-+
-+ rcu_read_lock();
-+ dev = dev_get_by_name_rcu(net, name);
-+ if (dev)
-+ dev_hold(dev);
-+ rcu_read_unlock();
-+ return dev;
-+}
-+EXPORT_SYMBOL(dev_get_by_name);
-+
-+/**
-+ * __dev_get_by_index - find a device by its ifindex
-+ * @net: the applicable net namespace
-+ * @ifindex: index of device
-+ *
-+ * Search for an interface by index. Returns %NULL if the device
-+ * is not found or a pointer to the device. The device has not
-+ * had its reference counter increased so the caller must be careful
-+ * about locking. The caller must hold either the RTNL semaphore
-+ * or @dev_base_lock.
-+ */
-+
-+struct net_device *__dev_get_by_index(struct net *net, int ifindex)
-+{
-+ struct net_device *dev;
-+ struct hlist_head *head = dev_index_hash(net, ifindex);
-+
-+ hlist_for_each_entry(dev, head, index_hlist)
-+ if (dev->ifindex == ifindex)
-+ return dev;
-+
-+ return NULL;
-+}
-+EXPORT_SYMBOL(__dev_get_by_index);
-+
-+/**
-+ * dev_get_by_index_rcu - find a device by its ifindex
-+ * @net: the applicable net namespace
-+ * @ifindex: index of device
-+ *
-+ * Search for an interface by index. Returns %NULL if the device
-+ * is not found or a pointer to the device. The device has not
-+ * had its reference counter increased so the caller must be careful
-+ * about locking. The caller must hold RCU lock.
-+ */
-+
-+struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
-+{
-+ struct net_device *dev;
-+ struct hlist_head *head = dev_index_hash(net, ifindex);
-+
-+ hlist_for_each_entry_rcu(dev, head, index_hlist)
-+ if (dev->ifindex == ifindex)
-+ return dev;
-+
-+ return NULL;
-+}
-+EXPORT_SYMBOL(dev_get_by_index_rcu);
-+
-+
-+/**
-+ * dev_get_by_index - find a device by its ifindex
-+ * @net: the applicable net namespace
-+ * @ifindex: index of device
-+ *
-+ * Search for an interface by index. Returns NULL if the device
-+ * is not found or a pointer to the device. The device returned has
-+ * had a reference added and the pointer is safe until the user calls
-+ * dev_put to indicate they have finished with it.
-+ */
-+
-+struct net_device *dev_get_by_index(struct net *net, int ifindex)
-+{
-+ struct net_device *dev;
-+
-+ rcu_read_lock();
-+ dev = dev_get_by_index_rcu(net, ifindex);
-+ if (dev)
-+ dev_hold(dev);
-+ rcu_read_unlock();
-+ return dev;
-+}
-+EXPORT_SYMBOL(dev_get_by_index);
-+
-+/**
-+ * netdev_get_name - get a netdevice name, knowing its ifindex.
-+ * @net: network namespace
-+ * @name: a pointer to the buffer where the name will be stored.
-+ * @ifindex: the ifindex of the interface to get the name from.
-+ *
-+ * The use of raw_seqcount_begin() and cond_resched() before
-+ * retrying is required as we want to give the writers a chance
-+ * to complete when CONFIG_PREEMPT is not set.
-+ */
-+int netdev_get_name(struct net *net, char *name, int ifindex)
-+{
-+ struct net_device *dev;
-+ unsigned int seq;
-+
-+retry:
-+ seq = raw_seqcount_begin(&devnet_rename_seq);
-+ rcu_read_lock();
-+ dev = dev_get_by_index_rcu(net, ifindex);
-+ if (!dev) {
-+ rcu_read_unlock();
-+ return -ENODEV;
-+ }
-+
-+ strcpy(name, dev->name);
-+ rcu_read_unlock();
-+ if (read_seqcount_retry(&devnet_rename_seq, seq)) {
-+ cond_resched();
-+ goto retry;
-+ }
-+
-+ return 0;
-+}
-+
-+/**
-+ * dev_getbyhwaddr_rcu - find a device by its hardware address
-+ * @net: the applicable net namespace
-+ * @type: media type of device
-+ * @ha: hardware address
-+ *
-+ * Search for an interface by MAC address. Returns NULL if the device
-+ * is not found or a pointer to the device.
-+ * The caller must hold RCU or RTNL.
-+ * The returned device has not had its ref count increased
-+ * and the caller must therefore be careful about locking
-+ *
-+ */
-+
-+struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
-+ const char *ha)
-+{
-+ struct net_device *dev;
-+
-+ for_each_netdev_rcu(net, dev)
-+ if (dev->type == type &&
-+ !memcmp(dev->dev_addr, ha, dev->addr_len))
-+ return dev;
-+
-+ return NULL;
-+}
-+EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
-+
-+struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
-+{
-+ struct net_device *dev;
-+
-+ ASSERT_RTNL();
-+ for_each_netdev(net, dev)
-+ if (dev->type == type)
-+ return dev;
-+
-+ return NULL;
-+}
-+EXPORT_SYMBOL(__dev_getfirstbyhwtype);
-+
-+struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
-+{
-+ struct net_device *dev, *ret = NULL;
-+
-+ rcu_read_lock();
-+ for_each_netdev_rcu(net, dev)
-+ if (dev->type == type) {
-+ dev_hold(dev);
-+ ret = dev;
-+ break;
-+ }
-+ rcu_read_unlock();
-+ return ret;
-+}
-+EXPORT_SYMBOL(dev_getfirstbyhwtype);
-+
-+/**
-+ * __dev_get_by_flags - find any device with given flags
-+ * @net: the applicable net namespace
-+ * @if_flags: IFF_* values
-+ * @mask: bitmask of bits in if_flags to check
-+ *
-+ * Search for any interface with the given flags. Returns NULL if a device
-+ * is not found or a pointer to the device. Must be called inside
-+ * rtnl_lock(), and result refcount is unchanged.
-+ */
-+
-+struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
-+ unsigned short mask)
-+{
-+ struct net_device *dev, *ret;
-+
-+ ASSERT_RTNL();
-+
-+ ret = NULL;
-+ for_each_netdev(net, dev) {
-+ if (((dev->flags ^ if_flags) & mask) == 0) {
-+ ret = dev;
-+ break;
-+ }
-+ }
-+ return ret;
-+}
-+EXPORT_SYMBOL(__dev_get_by_flags);
-+
-+/**
-+ * dev_valid_name - check if name is okay for network device
-+ * @name: name string
-+ *
-+ * Network device names need to be valid file names to
-+ * to allow sysfs to work. We also disallow any kind of
-+ * whitespace.
-+ */
-+bool dev_valid_name(const char *name)
-+{
-+ if (*name == '\0')
-+ return false;
-+ if (strlen(name) >= IFNAMSIZ)
-+ return false;
-+ if (!strcmp(name, ".") || !strcmp(name, ".."))
-+ return false;
-+
-+ while (*name) {
-+ if (*name == '/' || *name == ':' || isspace(*name))
-+ return false;
-+ name++;
-+ }
-+ return true;
-+}
-+EXPORT_SYMBOL(dev_valid_name);
-+
-+/**
-+ * __dev_alloc_name - allocate a name for a device
-+ * @net: network namespace to allocate the device name in
-+ * @name: name format string
-+ * @buf: scratch buffer and result name string
-+ *
-+ * Passed a format string - eg "lt%d" it will try and find a suitable
-+ * id. It scans list of devices to build up a free map, then chooses
-+ * the first empty slot. The caller must hold the dev_base or rtnl lock
-+ * while allocating the name and adding the device in order to avoid
-+ * duplicates.
-+ * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
-+ * Returns the number of the unit assigned or a negative errno code.
-+ */
-+
-+static int __dev_alloc_name(struct net *net, const char *name, char *buf)
-+{
-+ int i = 0;
-+ const char *p;
-+ const int max_netdevices = 8*PAGE_SIZE;
-+ unsigned long *inuse;
-+ struct net_device *d;
-+
-+ p = strnchr(name, IFNAMSIZ-1, '%');
-+ if (p) {
-+ /*
-+ * Verify the string as this thing may have come from
-+ * the user. There must be either one "%d" and no other "%"
-+ * characters.
-+ */
-+ if (p[1] != 'd' || strchr(p + 2, '%'))
-+ return -EINVAL;
-+
-+ /* Use one page as a bit array of possible slots */
-+ inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
-+ if (!inuse)
-+ return -ENOMEM;
-+
-+ for_each_netdev(net, d) {
-+ if (!sscanf(d->name, name, &i))
-+ continue;
-+ if (i < 0 || i >= max_netdevices)
-+ continue;
-+
-+ /* avoid cases where sscanf is not exact inverse of printf */
-+ snprintf(buf, IFNAMSIZ, name, i);
-+ if (!strncmp(buf, d->name, IFNAMSIZ))
-+ set_bit(i, inuse);
-+ }
-+
-+ i = find_first_zero_bit(inuse, max_netdevices);
-+ free_page((unsigned long) inuse);
-+ }
-+
-+ if (buf != name)
-+ snprintf(buf, IFNAMSIZ, name, i);
-+ if (!__dev_get_by_name(net, buf))
-+ return i;
-+
-+ /* It is possible to run out of possible slots
-+ * when the name is long and there isn't enough space left
-+ * for the digits, or if all bits are used.
-+ */
-+ return -ENFILE;
-+}
-+
-+/**
-+ * dev_alloc_name - allocate a name for a device
-+ * @dev: device
-+ * @name: name format string
-+ *
-+ * Passed a format string - eg "lt%d" it will try and find a suitable
-+ * id. It scans list of devices to build up a free map, then chooses
-+ * the first empty slot. The caller must hold the dev_base or rtnl lock
-+ * while allocating the name and adding the device in order to avoid
-+ * duplicates.
-+ * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
-+ * Returns the number of the unit assigned or a negative errno code.
-+ */
-+
-+int dev_alloc_name(struct net_device *dev, const char *name)
-+{
-+ char buf[IFNAMSIZ];
-+ struct net *net;
-+ int ret;
-+
-+ BUG_ON(!dev_net(dev));
-+ net = dev_net(dev);
-+ ret = __dev_alloc_name(net, name, buf);
-+ if (ret >= 0)
-+ strlcpy(dev->name, buf, IFNAMSIZ);
-+ return ret;
-+}
-+EXPORT_SYMBOL(dev_alloc_name);
-+
-+static int dev_alloc_name_ns(struct net *net,
-+ struct net_device *dev,
-+ const char *name)
-+{
-+ char buf[IFNAMSIZ];
-+ int ret;
-+
-+ ret = __dev_alloc_name(net, name, buf);
-+ if (ret >= 0)
-+ strlcpy(dev->name, buf, IFNAMSIZ);
-+ return ret;
-+}
-+
-+static int dev_get_valid_name(struct net *net,
-+ struct net_device *dev,
-+ const char *name)
-+{
-+ BUG_ON(!net);
-+
-+ if (!dev_valid_name(name))
-+ return -EINVAL;
-+
-+ if (strchr(name, '%'))
-+ return dev_alloc_name_ns(net, dev, name);
-+ else if (__dev_get_by_name(net, name))
-+ return -EEXIST;
-+ else if (dev->name != name)
-+ strlcpy(dev->name, name, IFNAMSIZ);
-+
-+ return 0;
-+}
-+
-+/**
-+ * dev_change_name - change name of a device
-+ * @dev: device
-+ * @newname: name (or format string) must be at least IFNAMSIZ
-+ *
-+ * Change name of a device, can pass format strings "eth%d".
-+ * for wildcarding.
-+ */
-+int dev_change_name(struct net_device *dev, const char *newname)
-+{
-+ unsigned char old_assign_type;
-+ char oldname[IFNAMSIZ];
-+ int err = 0;
-+ int ret;
-+ struct net *net;
-+
-+ ASSERT_RTNL();
-+ BUG_ON(!dev_net(dev));
-+
-+ net = dev_net(dev);
-+ if (dev->flags & IFF_UP)
-+ return -EBUSY;
-+
-+ write_seqcount_begin(&devnet_rename_seq);
-+
-+ if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
-+ write_seqcount_end(&devnet_rename_seq);
-+ return 0;
-+ }
-+
-+ memcpy(oldname, dev->name, IFNAMSIZ);
-+
-+ err = dev_get_valid_name(net, dev, newname);
-+ if (err < 0) {
-+ write_seqcount_end(&devnet_rename_seq);
-+ return err;
-+ }
-+
-+ if (oldname[0] && !strchr(oldname, '%'))
-+ netdev_info(dev, "renamed from %s\n", oldname);
-+
-+ old_assign_type = dev->name_assign_type;
-+ dev->name_assign_type = NET_NAME_RENAMED;
-+
-+rollback:
-+ ret = device_rename(&dev->dev, dev->name);
-+ if (ret) {
-+ memcpy(dev->name, oldname, IFNAMSIZ);
-+ dev->name_assign_type = old_assign_type;
-+ write_seqcount_end(&devnet_rename_seq);
-+ return ret;
-+ }
-+
-+ write_seqcount_end(&devnet_rename_seq);
-+
-+ netdev_adjacent_rename_links(dev, oldname);
-+
-+ write_lock_bh(&dev_base_lock);
-+ hlist_del_rcu(&dev->name_hlist);
-+ write_unlock_bh(&dev_base_lock);
-+
-+ synchronize_rcu();
-+
-+ write_lock_bh(&dev_base_lock);
-+ hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
-+ write_unlock_bh(&dev_base_lock);
-+
-+ ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
-+ ret = notifier_to_errno(ret);
-+
-+ if (ret) {
-+ /* err >= 0 after dev_alloc_name() or stores the first errno */
-+ if (err >= 0) {
-+ err = ret;
-+ write_seqcount_begin(&devnet_rename_seq);
-+ memcpy(dev->name, oldname, IFNAMSIZ);
-+ memcpy(oldname, newname, IFNAMSIZ);
-+ dev->name_assign_type = old_assign_type;
-+ old_assign_type = NET_NAME_RENAMED;
-+ goto rollback;
-+ } else {
-+ pr_err("%s: name change rollback failed: %d\n",
-+ dev->name, ret);
-+ }
-+ }
-+
-+ return err;
-+}
-+
-+/**
-+ * dev_set_alias - change ifalias of a device
-+ * @dev: device
-+ * @alias: name up to IFALIASZ
-+ * @len: limit of bytes to copy from info
-+ *
-+ * Set ifalias for a device,
-+ */
-+int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
-+{
-+ char *new_ifalias;
-+
-+ ASSERT_RTNL();
-+
-+ if (len >= IFALIASZ)
-+ return -EINVAL;
-+
-+ if (!len) {
-+ kfree(dev->ifalias);
-+ dev->ifalias = NULL;
-+ return 0;
-+ }
-+
-+ new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
-+ if (!new_ifalias)
-+ return -ENOMEM;
-+ dev->ifalias = new_ifalias;
-+
-+ strlcpy(dev->ifalias, alias, len+1);
-+ return len;
-+}
-+
-+
-+/**
-+ * netdev_features_change - device changes features
-+ * @dev: device to cause notification
-+ *
-+ * Called to indicate a device has changed features.
-+ */
-+void netdev_features_change(struct net_device *dev)
-+{
-+ call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
-+}
-+EXPORT_SYMBOL(netdev_features_change);
-+
-+/**
-+ * netdev_state_change - device changes state
-+ * @dev: device to cause notification
-+ *
-+ * Called to indicate a device has changed state. This function calls
-+ * the notifier chains for netdev_chain and sends a NEWLINK message
-+ * to the routing socket.
-+ */
-+void netdev_state_change(struct net_device *dev)
-+{
-+ if (dev->flags & IFF_UP) {
-+ struct netdev_notifier_change_info change_info;
-+
-+ change_info.flags_changed = 0;
-+ call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
-+ &change_info.info);
-+ rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
-+ }
-+}
-+EXPORT_SYMBOL(netdev_state_change);
-+
-+/**
-+ * netdev_notify_peers - notify network peers about existence of @dev
-+ * @dev: network device
-+ *
-+ * Generate traffic such that interested network peers are aware of
-+ * @dev, such as by generating a gratuitous ARP. This may be used when
-+ * a device wants to inform the rest of the network about some sort of
-+ * reconfiguration such as a failover event or virtual machine
-+ * migration.
-+ */
-+void netdev_notify_peers(struct net_device *dev)
-+{
-+ rtnl_lock();
-+ call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
-+ rtnl_unlock();
-+}
-+EXPORT_SYMBOL(netdev_notify_peers);
-+
-+static int __dev_open(struct net_device *dev)
-+{
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+ int ret;
-+
-+ ASSERT_RTNL();
-+
-+ if (!netif_device_present(dev))
-+ return -ENODEV;
-+
-+ /* Block netpoll from trying to do any rx path servicing.
-+ * If we don't do this there is a chance ndo_poll_controller
-+ * or ndo_poll may be running while we open the device
-+ */
-+ netpoll_poll_disable(dev);
-+
-+ ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
-+ ret = notifier_to_errno(ret);
-+ if (ret)
-+ return ret;
-+
-+ set_bit(__LINK_STATE_START, &dev->state);
-+
-+ if (ops->ndo_validate_addr)
-+ ret = ops->ndo_validate_addr(dev);
-+
-+ if (!ret && ops->ndo_open)
-+ ret = ops->ndo_open(dev);
-+
-+ netpoll_poll_enable(dev);
-+
-+ if (ret)
-+ clear_bit(__LINK_STATE_START, &dev->state);
-+ else {
-+ dev->flags |= IFF_UP;
-+ dev_set_rx_mode(dev);
-+ dev_activate(dev);
-+ add_device_randomness(dev->dev_addr, dev->addr_len);
-+ }
-+
-+ return ret;
-+}
-+
-+/**
-+ * dev_open - prepare an interface for use.
-+ * @dev: device to open
-+ *
-+ * Takes a device from down to up state. The device's private open
-+ * function is invoked and then the multicast lists are loaded. Finally
-+ * the device is moved into the up state and a %NETDEV_UP message is
-+ * sent to the netdev notifier chain.
-+ *
-+ * Calling this function on an active interface is a nop. On a failure
-+ * a negative errno code is returned.
-+ */
-+int dev_open(struct net_device *dev)
-+{
-+ int ret;
-+
-+ if (dev->flags & IFF_UP)
-+ return 0;
-+
-+ ret = __dev_open(dev);
-+ if (ret < 0)
-+ return ret;
-+
-+ rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
-+ call_netdevice_notifiers(NETDEV_UP, dev);
-+
-+ return ret;
-+}
-+EXPORT_SYMBOL(dev_open);
-+
-+static int __dev_close_many(struct list_head *head)
-+{
-+ struct net_device *dev;
-+
-+ ASSERT_RTNL();
-+ might_sleep();
-+
-+ list_for_each_entry(dev, head, close_list) {
-+ /* Temporarily disable netpoll until the interface is down */
-+ netpoll_poll_disable(dev);
-+
-+ call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
-+
-+ clear_bit(__LINK_STATE_START, &dev->state);
-+
-+ /* Synchronize to scheduled poll. We cannot touch poll list, it
-+ * can be even on different cpu. So just clear netif_running().
-+ *
-+ * dev->stop() will invoke napi_disable() on all of it's
-+ * napi_struct instances on this device.
-+ */
-+ smp_mb__after_atomic(); /* Commit netif_running(). */
-+ }
-+
-+ dev_deactivate_many(head);
-+
-+ list_for_each_entry(dev, head, close_list) {
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+
-+ /*
-+ * Call the device specific close. This cannot fail.
-+ * Only if device is UP
-+ *
-+ * We allow it to be called even after a DETACH hot-plug
-+ * event.
-+ */
-+ if (ops->ndo_stop)
-+ ops->ndo_stop(dev);
-+
-+ dev->flags &= ~IFF_UP;
-+ netpoll_poll_enable(dev);
-+ }
-+
-+ return 0;
-+}
-+
-+static int __dev_close(struct net_device *dev)
-+{
-+ int retval;
-+ LIST_HEAD(single);
-+
-+ list_add(&dev->close_list, &single);
-+ retval = __dev_close_many(&single);
-+ list_del(&single);
-+
-+ return retval;
-+}
-+
-+int dev_close_many(struct list_head *head, bool unlink)
-+{
-+ struct net_device *dev, *tmp;
-+
-+ /* Remove the devices that don't need to be closed */
-+ list_for_each_entry_safe(dev, tmp, head, close_list)
-+ if (!(dev->flags & IFF_UP))
-+ list_del_init(&dev->close_list);
-+
-+ __dev_close_many(head);
-+
-+ list_for_each_entry_safe(dev, tmp, head, close_list) {
-+ rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
-+ call_netdevice_notifiers(NETDEV_DOWN, dev);
-+ if (unlink)
-+ list_del_init(&dev->close_list);
-+ }
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL(dev_close_many);
-+
-+/**
-+ * dev_close - shutdown an interface.
-+ * @dev: device to shutdown
-+ *
-+ * This function moves an active device into down state. A
-+ * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
-+ * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
-+ * chain.
-+ */
-+int dev_close(struct net_device *dev)
-+{
-+ if (dev->flags & IFF_UP) {
-+ LIST_HEAD(single);
-+
-+ list_add(&dev->close_list, &single);
-+ dev_close_many(&single, true);
-+ list_del(&single);
-+ }
-+ return 0;
-+}
-+EXPORT_SYMBOL(dev_close);
-+
-+
-+/**
-+ * dev_disable_lro - disable Large Receive Offload on a device
-+ * @dev: device
-+ *
-+ * Disable Large Receive Offload (LRO) on a net device. Must be
-+ * called under RTNL. This is needed if received packets may be
-+ * forwarded to another interface.
-+ */
-+void dev_disable_lro(struct net_device *dev)
-+{
-+ struct net_device *lower_dev;
-+ struct list_head *iter;
-+
-+ dev->wanted_features &= ~NETIF_F_LRO;
-+ netdev_update_features(dev);
-+
-+ if (unlikely(dev->features & NETIF_F_LRO))
-+ netdev_WARN(dev, "failed to disable LRO!\n");
-+
-+ netdev_for_each_lower_dev(dev, lower_dev, iter)
-+ dev_disable_lro(lower_dev);
-+}
-+EXPORT_SYMBOL(dev_disable_lro);
-+
-+static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
-+ struct net_device *dev)
-+{
-+ struct netdev_notifier_info info;
-+
-+ netdev_notifier_info_init(&info, dev);
-+ return nb->notifier_call(nb, val, &info);
-+}
-+
-+static int dev_boot_phase = 1;
-+
-+/**
-+ * register_netdevice_notifier - register a network notifier block
-+ * @nb: notifier
-+ *
-+ * Register a notifier to be called when network device events occur.
-+ * The notifier passed is linked into the kernel structures and must
-+ * not be reused until it has been unregistered. A negative errno code
-+ * is returned on a failure.
-+ *
-+ * When registered all registration and up events are replayed
-+ * to the new notifier to allow device to have a race free
-+ * view of the network device list.
-+ */
-+
-+int register_netdevice_notifier(struct notifier_block *nb)
-+{
-+ struct net_device *dev;
-+ struct net_device *last;
-+ struct net *net;
-+ int err;
-+
-+ rtnl_lock();
-+ err = raw_notifier_chain_register(&netdev_chain, nb);
-+ if (err)
-+ goto unlock;
-+ if (dev_boot_phase)
-+ goto unlock;
-+ for_each_net(net) {
-+ for_each_netdev(net, dev) {
-+ err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
-+ err = notifier_to_errno(err);
-+ if (err)
-+ goto rollback;
-+
-+ if (!(dev->flags & IFF_UP))
-+ continue;
-+
-+ call_netdevice_notifier(nb, NETDEV_UP, dev);
-+ }
-+ }
-+
-+unlock:
-+ rtnl_unlock();
-+ return err;
-+
-+rollback:
-+ last = dev;
-+ for_each_net(net) {
-+ for_each_netdev(net, dev) {
-+ if (dev == last)
-+ goto outroll;
-+
-+ if (dev->flags & IFF_UP) {
-+ call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
-+ dev);
-+ call_netdevice_notifier(nb, NETDEV_DOWN, dev);
-+ }
-+ call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
-+ }
-+ }
-+
-+outroll:
-+ raw_notifier_chain_unregister(&netdev_chain, nb);
-+ goto unlock;
-+}
-+EXPORT_SYMBOL(register_netdevice_notifier);
-+
-+/**
-+ * unregister_netdevice_notifier - unregister a network notifier block
-+ * @nb: notifier
-+ *
-+ * Unregister a notifier previously registered by
-+ * register_netdevice_notifier(). The notifier is unlinked into the
-+ * kernel structures and may then be reused. A negative errno code
-+ * is returned on a failure.
-+ *
-+ * After unregistering unregister and down device events are synthesized
-+ * for all devices on the device list to the removed notifier to remove
-+ * the need for special case cleanup code.
-+ */
-+
-+int unregister_netdevice_notifier(struct notifier_block *nb)
-+{
-+ struct net_device *dev;
-+ struct net *net;
-+ int err;
-+
-+ rtnl_lock();
-+ err = raw_notifier_chain_unregister(&netdev_chain, nb);
-+ if (err)
-+ goto unlock;
-+
-+ for_each_net(net) {
-+ for_each_netdev(net, dev) {
-+ if (dev->flags & IFF_UP) {
-+ call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
-+ dev);
-+ call_netdevice_notifier(nb, NETDEV_DOWN, dev);
-+ }
-+ call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
-+ }
-+ }
-+unlock:
-+ rtnl_unlock();
-+ return err;
-+}
-+EXPORT_SYMBOL(unregister_netdevice_notifier);
-+
-+/**
-+ * call_netdevice_notifiers_info - call all network notifier blocks
-+ * @val: value passed unmodified to notifier function
-+ * @dev: net_device pointer passed unmodified to notifier function
-+ * @info: notifier information data
-+ *
-+ * Call all network notifier blocks. Parameters and return value
-+ * are as for raw_notifier_call_chain().
-+ */
-+
-+static int call_netdevice_notifiers_info(unsigned long val,
-+ struct net_device *dev,
-+ struct netdev_notifier_info *info)
-+{
-+ ASSERT_RTNL();
-+ netdev_notifier_info_init(info, dev);
-+ return raw_notifier_call_chain(&netdev_chain, val, info);
-+}
-+
-+/**
-+ * call_netdevice_notifiers - call all network notifier blocks
-+ * @val: value passed unmodified to notifier function
-+ * @dev: net_device pointer passed unmodified to notifier function
-+ *
-+ * Call all network notifier blocks. Parameters and return value
-+ * are as for raw_notifier_call_chain().
-+ */
-+
-+int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
-+{
-+ struct netdev_notifier_info info;
-+
-+ return call_netdevice_notifiers_info(val, dev, &info);
-+}
-+EXPORT_SYMBOL(call_netdevice_notifiers);
-+
-+#ifdef CONFIG_NET_CLS_ACT
-+static struct static_key ingress_needed __read_mostly;
-+
-+void net_inc_ingress_queue(void)
-+{
-+ static_key_slow_inc(&ingress_needed);
-+}
-+EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
-+
-+void net_dec_ingress_queue(void)
-+{
-+ static_key_slow_dec(&ingress_needed);
-+}
-+EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
-+#endif
-+
-+static struct static_key netstamp_needed __read_mostly;
-+#ifdef HAVE_JUMP_LABEL
-+/* We are not allowed to call static_key_slow_dec() from irq context
-+ * If net_disable_timestamp() is called from irq context, defer the
-+ * static_key_slow_dec() calls.
-+ */
-+static atomic_t netstamp_needed_deferred;
-+#endif
-+
-+void net_enable_timestamp(void)
-+{
-+#ifdef HAVE_JUMP_LABEL
-+ int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
-+
-+ if (deferred) {
-+ while (--deferred)
-+ static_key_slow_dec(&netstamp_needed);
-+ return;
-+ }
-+#endif
-+ static_key_slow_inc(&netstamp_needed);
-+}
-+EXPORT_SYMBOL(net_enable_timestamp);
-+
-+void net_disable_timestamp(void)
-+{
-+#ifdef HAVE_JUMP_LABEL
-+ if (in_interrupt()) {
-+ atomic_inc(&netstamp_needed_deferred);
-+ return;
-+ }
-+#endif
-+ static_key_slow_dec(&netstamp_needed);
-+}
-+EXPORT_SYMBOL(net_disable_timestamp);
-+
-+static inline void net_timestamp_set(struct sk_buff *skb)
-+{
-+ skb->tstamp.tv64 = 0;
-+ if (static_key_false(&netstamp_needed))
-+ __net_timestamp(skb);
-+}
-+
-+#define net_timestamp_check(COND, SKB) \
-+ if (static_key_false(&netstamp_needed)) { \
-+ if ((COND) && !(SKB)->tstamp.tv64) \
-+ __net_timestamp(SKB); \
-+ } \
-+
-+bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
-+{
-+ unsigned int len;
-+
-+ if (!(dev->flags & IFF_UP))
-+ return false;
-+
-+ len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
-+ if (skb->len <= len)
-+ return true;
-+
-+ /* if TSO is enabled, we don't care about the length as the packet
-+ * could be forwarded without being segmented before
-+ */
-+ if (skb_is_gso(skb))
-+ return true;
-+
-+ return false;
-+}
-+EXPORT_SYMBOL_GPL(is_skb_forwardable);
-+
-+int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
-+{
-+ if (skb_orphan_frags(skb, GFP_ATOMIC) ||
-+ unlikely(!is_skb_forwardable(dev, skb))) {
-+ atomic_long_inc(&dev->rx_dropped);
-+ kfree_skb(skb);
-+ return NET_RX_DROP;
-+ }
-+
-+ skb_scrub_packet(skb, true);
-+ skb->priority = 0;
-+ skb->protocol = eth_type_trans(skb, dev);
-+ skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL_GPL(__dev_forward_skb);
-+
-+/**
-+ * dev_forward_skb - loopback an skb to another netif
-+ *
-+ * @dev: destination network device
-+ * @skb: buffer to forward
-+ *
-+ * return values:
-+ * NET_RX_SUCCESS (no congestion)
-+ * NET_RX_DROP (packet was dropped, but freed)
-+ *
-+ * dev_forward_skb can be used for injecting an skb from the
-+ * start_xmit function of one device into the receive queue
-+ * of another device.
-+ *
-+ * The receiving device may be in another namespace, so
-+ * we have to clear all information in the skb that could
-+ * impact namespace isolation.
-+ */
-+int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
-+{
-+ return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
-+}
-+EXPORT_SYMBOL_GPL(dev_forward_skb);
-+
-+static inline int deliver_skb(struct sk_buff *skb,
-+ struct packet_type *pt_prev,
-+ struct net_device *orig_dev)
-+{
-+ if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
-+ return -ENOMEM;
-+ atomic_inc(&skb->users);
-+ return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
-+}
-+
-+static inline void deliver_ptype_list_skb(struct sk_buff *skb,
-+ struct packet_type **pt,
-+ struct net_device *orig_dev,
-+ __be16 type,
-+ struct list_head *ptype_list)
-+{
-+ struct packet_type *ptype, *pt_prev = *pt;
-+
-+ list_for_each_entry_rcu(ptype, ptype_list, list) {
-+ if (ptype->type != type)
-+ continue;
-+ if (pt_prev)
-+ deliver_skb(skb, pt_prev, orig_dev);
-+ pt_prev = ptype;
-+ }
-+ *pt = pt_prev;
-+}
-+
-+static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
-+{
-+ if (!ptype->af_packet_priv || !skb->sk)
-+ return false;
-+
-+ if (ptype->id_match)
-+ return ptype->id_match(ptype, skb->sk);
-+ else if ((struct sock *)ptype->af_packet_priv == skb->sk)
-+ return true;
-+
-+ return false;
-+}
-+
-+/*
-+ * Support routine. Sends outgoing frames to any network
-+ * taps currently in use.
-+ */
-+
-+static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
-+{
-+ struct packet_type *ptype;
-+ struct sk_buff *skb2 = NULL;
-+ struct packet_type *pt_prev = NULL;
-+ struct list_head *ptype_list = &ptype_all;
-+
-+ rcu_read_lock();
-+again:
-+ list_for_each_entry_rcu(ptype, ptype_list, list) {
-+ /* Never send packets back to the socket
-+ * they originated from - MvS (miquels@drinkel.ow.org)
-+ */
-+ if (skb_loop_sk(ptype, skb))
-+ continue;
-+
-+ if (pt_prev) {
-+ deliver_skb(skb2, pt_prev, skb->dev);
-+ pt_prev = ptype;
-+ continue;
-+ }
-+
-+ /* need to clone skb, done only once */
-+ skb2 = skb_clone(skb, GFP_ATOMIC);
-+ if (!skb2)
-+ goto out_unlock;
-+
-+ net_timestamp_set(skb2);
-+
-+ /* skb->nh should be correctly
-+ * set by sender, so that the second statement is
-+ * just protection against buggy protocols.
-+ */
-+ skb_reset_mac_header(skb2);
-+
-+ if (skb_network_header(skb2) < skb2->data ||
-+ skb_network_header(skb2) > skb_tail_pointer(skb2)) {
-+ net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
-+ ntohs(skb2->protocol),
-+ dev->name);
-+ skb_reset_network_header(skb2);
-+ }
-+
-+ skb2->transport_header = skb2->network_header;
-+ skb2->pkt_type = PACKET_OUTGOING;
-+ pt_prev = ptype;
-+ }
-+
-+ if (ptype_list == &ptype_all) {
-+ ptype_list = &dev->ptype_all;
-+ goto again;
-+ }
-+out_unlock:
-+ if (pt_prev)
-+ pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
-+ rcu_read_unlock();
-+}
-+
-+/**
-+ * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
-+ * @dev: Network device
-+ * @txq: number of queues available
-+ *
-+ * If real_num_tx_queues is changed the tc mappings may no longer be
-+ * valid. To resolve this verify the tc mapping remains valid and if
-+ * not NULL the mapping. With no priorities mapping to this
-+ * offset/count pair it will no longer be used. In the worst case TC0
-+ * is invalid nothing can be done so disable priority mappings. If is
-+ * expected that drivers will fix this mapping if they can before
-+ * calling netif_set_real_num_tx_queues.
-+ */
-+static void netif_setup_tc(struct net_device *dev, unsigned int txq)
-+{
-+ int i;
-+ struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
-+
-+ /* If TC0 is invalidated disable TC mapping */
-+ if (tc->offset + tc->count > txq) {
-+ pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
-+ dev->num_tc = 0;
-+ return;
-+ }
-+
-+ /* Invalidated prio to tc mappings set to TC0 */
-+ for (i = 1; i < TC_BITMASK + 1; i++) {
-+ int q = netdev_get_prio_tc_map(dev, i);
-+
-+ tc = &dev->tc_to_txq[q];
-+ if (tc->offset + tc->count > txq) {
-+ pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
-+ i, q);
-+ netdev_set_prio_tc_map(dev, i, 0);
-+ }
-+ }
-+}
-+
-+#ifdef CONFIG_XPS
-+static DEFINE_MUTEX(xps_map_mutex);
-+#define xmap_dereference(P) \
-+ rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
-+
-+static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
-+ int cpu, u16 index)
-+{
-+ struct xps_map *map = NULL;
-+ int pos;
-+
-+ if (dev_maps)
-+ map = xmap_dereference(dev_maps->cpu_map[cpu]);
-+
-+ for (pos = 0; map && pos < map->len; pos++) {
-+ if (map->queues[pos] == index) {
-+ if (map->len > 1) {
-+ map->queues[pos] = map->queues[--map->len];
-+ } else {
-+ RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
-+ kfree_rcu(map, rcu);
-+ map = NULL;
-+ }
-+ break;
-+ }
-+ }
-+
-+ return map;
-+}
-+
-+static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
-+{
-+ struct xps_dev_maps *dev_maps;
-+ int cpu, i;
-+ bool active = false;
-+
-+ mutex_lock(&xps_map_mutex);
-+ dev_maps = xmap_dereference(dev->xps_maps);
-+
-+ if (!dev_maps)
-+ goto out_no_maps;
-+
-+ for_each_possible_cpu(cpu) {
-+ for (i = index; i < dev->num_tx_queues; i++) {
-+ if (!remove_xps_queue(dev_maps, cpu, i))
-+ break;
-+ }
-+ if (i == dev->num_tx_queues)
-+ active = true;
-+ }
-+
-+ if (!active) {
-+ RCU_INIT_POINTER(dev->xps_maps, NULL);
-+ kfree_rcu(dev_maps, rcu);
-+ }
-+
-+ for (i = index; i < dev->num_tx_queues; i++)
-+ netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
-+ NUMA_NO_NODE);
-+
-+out_no_maps:
-+ mutex_unlock(&xps_map_mutex);
-+}
-+
-+static struct xps_map *expand_xps_map(struct xps_map *map,
-+ int cpu, u16 index)
-+{
-+ struct xps_map *new_map;
-+ int alloc_len = XPS_MIN_MAP_ALLOC;
-+ int i, pos;
-+
-+ for (pos = 0; map && pos < map->len; pos++) {
-+ if (map->queues[pos] != index)
-+ continue;
-+ return map;
-+ }
-+
-+ /* Need to add queue to this CPU's existing map */
-+ if (map) {
-+ if (pos < map->alloc_len)
-+ return map;
-+
-+ alloc_len = map->alloc_len * 2;
-+ }
-+
-+ /* Need to allocate new map to store queue on this CPU's map */
-+ new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
-+ cpu_to_node(cpu));
-+ if (!new_map)
-+ return NULL;
-+
-+ for (i = 0; i < pos; i++)
-+ new_map->queues[i] = map->queues[i];
-+ new_map->alloc_len = alloc_len;
-+ new_map->len = pos;
-+
-+ return new_map;
-+}
-+
-+int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
-+ u16 index)
-+{
-+ struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
-+ struct xps_map *map, *new_map;
-+ int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
-+ int cpu, numa_node_id = -2;
-+ bool active = false;
-+
-+ mutex_lock(&xps_map_mutex);
-+
-+ dev_maps = xmap_dereference(dev->xps_maps);
-+
-+ /* allocate memory for queue storage */
-+ for_each_online_cpu(cpu) {
-+ if (!cpumask_test_cpu(cpu, mask))
-+ continue;
-+
-+ if (!new_dev_maps)
-+ new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
-+ if (!new_dev_maps) {
-+ mutex_unlock(&xps_map_mutex);
-+ return -ENOMEM;
-+ }
-+
-+ map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
-+ NULL;
-+
-+ map = expand_xps_map(map, cpu, index);
-+ if (!map)
-+ goto error;
-+
-+ RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
-+ }
-+
-+ if (!new_dev_maps)
-+ goto out_no_new_maps;
-+
-+ for_each_possible_cpu(cpu) {
-+ if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
-+ /* add queue to CPU maps */
-+ int pos = 0;
-+
-+ map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
-+ while ((pos < map->len) && (map->queues[pos] != index))
-+ pos++;
-+
-+ if (pos == map->len)
-+ map->queues[map->len++] = index;
-+#ifdef CONFIG_NUMA
-+ if (numa_node_id == -2)
-+ numa_node_id = cpu_to_node(cpu);
-+ else if (numa_node_id != cpu_to_node(cpu))
-+ numa_node_id = -1;
-+#endif
-+ } else if (dev_maps) {
-+ /* fill in the new device map from the old device map */
-+ map = xmap_dereference(dev_maps->cpu_map[cpu]);
-+ RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
-+ }
-+
-+ }
-+
-+ rcu_assign_pointer(dev->xps_maps, new_dev_maps);
-+
-+ /* Cleanup old maps */
-+ if (dev_maps) {
-+ for_each_possible_cpu(cpu) {
-+ new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
-+ map = xmap_dereference(dev_maps->cpu_map[cpu]);
-+ if (map && map != new_map)
-+ kfree_rcu(map, rcu);
-+ }
-+
-+ kfree_rcu(dev_maps, rcu);
-+ }
-+
-+ dev_maps = new_dev_maps;
-+ active = true;
-+
-+out_no_new_maps:
-+ /* update Tx queue numa node */
-+ netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
-+ (numa_node_id >= 0) ? numa_node_id :
-+ NUMA_NO_NODE);
-+
-+ if (!dev_maps)
-+ goto out_no_maps;
-+
-+ /* removes queue from unused CPUs */
-+ for_each_possible_cpu(cpu) {
-+ if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
-+ continue;
-+
-+ if (remove_xps_queue(dev_maps, cpu, index))
-+ active = true;
-+ }
-+
-+ /* free map if not active */
-+ if (!active) {
-+ RCU_INIT_POINTER(dev->xps_maps, NULL);
-+ kfree_rcu(dev_maps, rcu);
-+ }
-+
-+out_no_maps:
-+ mutex_unlock(&xps_map_mutex);
-+
-+ return 0;
-+error:
-+ /* remove any maps that we added */
-+ for_each_possible_cpu(cpu) {
-+ new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
-+ map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
-+ NULL;
-+ if (new_map && new_map != map)
-+ kfree(new_map);
-+ }
-+
-+ mutex_unlock(&xps_map_mutex);
-+
-+ kfree(new_dev_maps);
-+ return -ENOMEM;
-+}
-+EXPORT_SYMBOL(netif_set_xps_queue);
-+
-+#endif
-+/*
-+ * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
-+ * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
-+ */
-+int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
-+{
-+ int rc;
-+
-+ if (txq < 1 || txq > dev->num_tx_queues)
-+ return -EINVAL;
-+
-+ if (dev->reg_state == NETREG_REGISTERED ||
-+ dev->reg_state == NETREG_UNREGISTERING) {
-+ ASSERT_RTNL();
-+
-+ rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
-+ txq);
-+ if (rc)
-+ return rc;
-+
-+ if (dev->num_tc)
-+ netif_setup_tc(dev, txq);
-+
-+ if (txq < dev->real_num_tx_queues) {
-+ qdisc_reset_all_tx_gt(dev, txq);
-+#ifdef CONFIG_XPS
-+ netif_reset_xps_queues_gt(dev, txq);
-+#endif
-+ }
-+ }
-+
-+ dev->real_num_tx_queues = txq;
-+ return 0;
-+}
-+EXPORT_SYMBOL(netif_set_real_num_tx_queues);
-+
-+#ifdef CONFIG_SYSFS
-+/**
-+ * netif_set_real_num_rx_queues - set actual number of RX queues used
-+ * @dev: Network device
-+ * @rxq: Actual number of RX queues
-+ *
-+ * This must be called either with the rtnl_lock held or before
-+ * registration of the net device. Returns 0 on success, or a
-+ * negative error code. If called before registration, it always
-+ * succeeds.
-+ */
-+int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
-+{
-+ int rc;
-+
-+ if (rxq < 1 || rxq > dev->num_rx_queues)
-+ return -EINVAL;
-+
-+ if (dev->reg_state == NETREG_REGISTERED) {
-+ ASSERT_RTNL();
-+
-+ rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
-+ rxq);
-+ if (rc)
-+ return rc;
-+ }
-+
-+ dev->real_num_rx_queues = rxq;
-+ return 0;
-+}
-+EXPORT_SYMBOL(netif_set_real_num_rx_queues);
-+#endif
-+
-+/**
-+ * netif_get_num_default_rss_queues - default number of RSS queues
-+ *
-+ * This routine should set an upper limit on the number of RSS queues
-+ * used by default by multiqueue devices.
-+ */
-+int netif_get_num_default_rss_queues(void)
-+{
-+ return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
-+}
-+EXPORT_SYMBOL(netif_get_num_default_rss_queues);
-+
-+static inline void __netif_reschedule(struct Qdisc *q)
-+{
-+ struct softnet_data *sd;
-+ unsigned long flags;
-+
-+ local_irq_save(flags);
-+ sd = this_cpu_ptr(&softnet_data);
-+ q->next_sched = NULL;
-+ *sd->output_queue_tailp = q;
-+ sd->output_queue_tailp = &q->next_sched;
-+ raise_softirq_irqoff(NET_TX_SOFTIRQ);
-+ local_irq_restore(flags);
-+}
-+
-+void __netif_schedule(struct Qdisc *q)
-+{
-+ if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
-+ __netif_reschedule(q);
-+}
-+EXPORT_SYMBOL(__netif_schedule);
-+
-+struct dev_kfree_skb_cb {
-+ enum skb_free_reason reason;
-+};
-+
-+static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
-+{
-+ return (struct dev_kfree_skb_cb *)skb->cb;
-+}
-+
-+void netif_schedule_queue(struct netdev_queue *txq)
-+{
-+ rcu_read_lock();
-+ if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
-+ struct Qdisc *q = rcu_dereference(txq->qdisc);
-+
-+ __netif_schedule(q);
-+ }
-+ rcu_read_unlock();
-+}
-+EXPORT_SYMBOL(netif_schedule_queue);
-+
-+/**
-+ * netif_wake_subqueue - allow sending packets on subqueue
-+ * @dev: network device
-+ * @queue_index: sub queue index
-+ *
-+ * Resume individual transmit queue of a device with multiple transmit queues.
-+ */
-+void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
-+{
-+ struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
-+
-+ if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
-+ struct Qdisc *q;
-+
-+ rcu_read_lock();
-+ q = rcu_dereference(txq->qdisc);
-+ __netif_schedule(q);
-+ rcu_read_unlock();
-+ }
-+}
-+EXPORT_SYMBOL(netif_wake_subqueue);
-+
-+void netif_tx_wake_queue(struct netdev_queue *dev_queue)
-+{
-+ if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
-+ struct Qdisc *q;
-+
-+ rcu_read_lock();
-+ q = rcu_dereference(dev_queue->qdisc);
-+ __netif_schedule(q);
-+ rcu_read_unlock();
-+ }
-+}
-+EXPORT_SYMBOL(netif_tx_wake_queue);
-+
-+void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
-+{
-+ unsigned long flags;
-+
-+ if (likely(atomic_read(&skb->users) == 1)) {
-+ smp_rmb();
-+ atomic_set(&skb->users, 0);
-+ } else if (likely(!atomic_dec_and_test(&skb->users))) {
-+ return;
-+ }
-+ get_kfree_skb_cb(skb)->reason = reason;
-+ local_irq_save(flags);
-+ skb->next = __this_cpu_read(softnet_data.completion_queue);
-+ __this_cpu_write(softnet_data.completion_queue, skb);
-+ raise_softirq_irqoff(NET_TX_SOFTIRQ);
-+ local_irq_restore(flags);
-+}
-+EXPORT_SYMBOL(__dev_kfree_skb_irq);
-+
-+void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
-+{
-+ if (in_irq() || irqs_disabled())
-+ __dev_kfree_skb_irq(skb, reason);
-+ else
-+ dev_kfree_skb(skb);
-+}
-+EXPORT_SYMBOL(__dev_kfree_skb_any);
-+
-+
-+/**
-+ * netif_device_detach - mark device as removed
-+ * @dev: network device
-+ *
-+ * Mark device as removed from system and therefore no longer available.
-+ */
-+void netif_device_detach(struct net_device *dev)
-+{
-+ if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
-+ netif_running(dev)) {
-+ netif_tx_stop_all_queues(dev);
-+ }
-+}
-+EXPORT_SYMBOL(netif_device_detach);
-+
-+/**
-+ * netif_device_attach - mark device as attached
-+ * @dev: network device
-+ *
-+ * Mark device as attached from system and restart if needed.
-+ */
-+void netif_device_attach(struct net_device *dev)
-+{
-+ if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
-+ netif_running(dev)) {
-+ netif_tx_wake_all_queues(dev);
-+ __netdev_watchdog_up(dev);
-+ }
-+}
-+EXPORT_SYMBOL(netif_device_attach);
-+
-+static void skb_warn_bad_offload(const struct sk_buff *skb)
-+{
-+ static const netdev_features_t null_features = 0;
-+ struct net_device *dev = skb->dev;
-+ const char *driver = "";
-+
-+ if (!net_ratelimit())
-+ return;
-+
-+ if (dev && dev->dev.parent)
-+ driver = dev_driver_string(dev->dev.parent);
-+
-+ WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
-+ "gso_type=%d ip_summed=%d\n",
-+ driver, dev ? &dev->features : &null_features,
-+ skb->sk ? &skb->sk->sk_route_caps : &null_features,
-+ skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
-+ skb_shinfo(skb)->gso_type, skb->ip_summed);
-+}
-+
-+/*
-+ * Invalidate hardware checksum when packet is to be mangled, and
-+ * complete checksum manually on outgoing path.
-+ */
-+int skb_checksum_help(struct sk_buff *skb)
-+{
-+ __wsum csum;
-+ int ret = 0, offset;
-+
-+ if (skb->ip_summed == CHECKSUM_COMPLETE)
-+ goto out_set_summed;
-+
-+ if (unlikely(skb_shinfo(skb)->gso_size)) {
-+ skb_warn_bad_offload(skb);
-+ return -EINVAL;
-+ }
-+
-+ /* Before computing a checksum, we should make sure no frag could
-+ * be modified by an external entity : checksum could be wrong.
-+ */
-+ if (skb_has_shared_frag(skb)) {
-+ ret = __skb_linearize(skb);
-+ if (ret)
-+ goto out;
-+ }
-+
-+ offset = skb_checksum_start_offset(skb);
-+ BUG_ON(offset >= skb_headlen(skb));
-+ csum = skb_checksum(skb, offset, skb->len - offset, 0);
-+
-+ offset += skb->csum_offset;
-+ BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
-+
-+ if (skb_cloned(skb) &&
-+ !skb_clone_writable(skb, offset + sizeof(__sum16))) {
-+ ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
-+ if (ret)
-+ goto out;
-+ }
-+
-+ *(__sum16 *)(skb->data + offset) = csum_fold(csum);
-+out_set_summed:
-+ skb->ip_summed = CHECKSUM_NONE;
-+out:
-+ return ret;
-+}
-+EXPORT_SYMBOL(skb_checksum_help);
-+
-+__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
-+{
-+ __be16 type = skb->protocol;
-+
-+ /* Tunnel gso handlers can set protocol to ethernet. */
-+ if (type == htons(ETH_P_TEB)) {
-+ struct ethhdr *eth;
-+
-+ if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
-+ return 0;
-+
-+ eth = (struct ethhdr *)skb_mac_header(skb);
-+ type = eth->h_proto;
-+ }
-+
-+ return __vlan_get_protocol(skb, type, depth);
-+}
-+
-+/**
-+ * skb_mac_gso_segment - mac layer segmentation handler.
-+ * @skb: buffer to segment
-+ * @features: features for the output path (see dev->features)
-+ */
-+struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
-+ netdev_features_t features)
-+{
-+ struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
-+ struct packet_offload *ptype;
-+ int vlan_depth = skb->mac_len;
-+ __be16 type = skb_network_protocol(skb, &vlan_depth);
-+
-+ if (unlikely(!type))
-+ return ERR_PTR(-EINVAL);
-+
-+ __skb_pull(skb, vlan_depth);
-+
-+ rcu_read_lock();
-+ list_for_each_entry_rcu(ptype, &offload_base, list) {
-+ if (ptype->type == type && ptype->callbacks.gso_segment) {
-+ segs = ptype->callbacks.gso_segment(skb, features);
-+ break;
-+ }
-+ }
-+ rcu_read_unlock();
-+
-+ __skb_push(skb, skb->data - skb_mac_header(skb));
-+
-+ return segs;
-+}
-+EXPORT_SYMBOL(skb_mac_gso_segment);
-+
-+
-+/* openvswitch calls this on rx path, so we need a different check.
-+ */
-+static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
-+{
-+ if (tx_path)
-+ return skb->ip_summed != CHECKSUM_PARTIAL;
-+ else
-+ return skb->ip_summed == CHECKSUM_NONE;
-+}
-+
-+/**
-+ * __skb_gso_segment - Perform segmentation on skb.
-+ * @skb: buffer to segment
-+ * @features: features for the output path (see dev->features)
-+ * @tx_path: whether it is called in TX path
-+ *
-+ * This function segments the given skb and returns a list of segments.
-+ *
-+ * It may return NULL if the skb requires no segmentation. This is
-+ * only possible when GSO is used for verifying header integrity.
-+ */
-+struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
-+ netdev_features_t features, bool tx_path)
-+{
-+ if (unlikely(skb_needs_check(skb, tx_path))) {
-+ int err;
-+
-+ skb_warn_bad_offload(skb);
-+
-+ err = skb_cow_head(skb, 0);
-+ if (err < 0)
-+ return ERR_PTR(err);
-+ }
-+
-+ SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
-+ SKB_GSO_CB(skb)->encap_level = 0;
-+
-+ skb_reset_mac_header(skb);
-+ skb_reset_mac_len(skb);
-+
-+ return skb_mac_gso_segment(skb, features);
-+}
-+EXPORT_SYMBOL(__skb_gso_segment);
-+
-+/* Take action when hardware reception checksum errors are detected. */
-+#ifdef CONFIG_BUG
-+void netdev_rx_csum_fault(struct net_device *dev)
-+{
-+ if (net_ratelimit()) {
-+ pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
-+ dump_stack();
-+ }
-+}
-+EXPORT_SYMBOL(netdev_rx_csum_fault);
-+#endif
-+
-+/* Actually, we should eliminate this check as soon as we know, that:
-+ * 1. IOMMU is present and allows to map all the memory.
-+ * 2. No high memory really exists on this machine.
-+ */
-+
-+static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
-+{
-+#ifdef CONFIG_HIGHMEM
-+ int i;
-+ if (!(dev->features & NETIF_F_HIGHDMA)) {
-+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
-+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
-+ if (PageHighMem(skb_frag_page(frag)))
-+ return 1;
-+ }
-+ }
-+
-+ if (PCI_DMA_BUS_IS_PHYS) {
-+ struct device *pdev = dev->dev.parent;
-+
-+ if (!pdev)
-+ return 0;
-+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
-+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
-+ dma_addr_t addr = page_to_phys(skb_frag_page(frag));
-+ if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
-+ return 1;
-+ }
-+ }
-+#endif
-+ return 0;
-+}
-+
-+/* If MPLS offload request, verify we are testing hardware MPLS features
-+ * instead of standard features for the netdev.
-+ */
-+#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
-+static netdev_features_t net_mpls_features(struct sk_buff *skb,
-+ netdev_features_t features,
-+ __be16 type)
-+{
-+ if (eth_p_mpls(type))
-+ features &= skb->dev->mpls_features;
-+
-+ return features;
-+}
-+#else
-+static netdev_features_t net_mpls_features(struct sk_buff *skb,
-+ netdev_features_t features,
-+ __be16 type)
-+{
-+ return features;
-+}
-+#endif
-+
-+static netdev_features_t harmonize_features(struct sk_buff *skb,
-+ netdev_features_t features)
-+{
-+ int tmp;
-+ __be16 type;
-+
-+ type = skb_network_protocol(skb, &tmp);
-+ features = net_mpls_features(skb, features, type);
-+
-+ if (skb->ip_summed != CHECKSUM_NONE &&
-+ !can_checksum_protocol(features, type)) {
-+ features &= ~NETIF_F_ALL_CSUM;
-+ } else if (illegal_highdma(skb->dev, skb)) {
-+ features &= ~NETIF_F_SG;
-+ }
-+
-+ return features;
-+}
-+
-+netdev_features_t passthru_features_check(struct sk_buff *skb,
-+ struct net_device *dev,
-+ netdev_features_t features)
-+{
-+ return features;
-+}
-+EXPORT_SYMBOL(passthru_features_check);
-+
-+static netdev_features_t dflt_features_check(const struct sk_buff *skb,
-+ struct net_device *dev,
-+ netdev_features_t features)
-+{
-+ return vlan_features_check(skb, features);
-+}
-+
-+netdev_features_t netif_skb_features(struct sk_buff *skb)
-+{
-+ struct net_device *dev = skb->dev;
-+ netdev_features_t features = dev->features;
-+ u16 gso_segs = skb_shinfo(skb)->gso_segs;
-+
-+ if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
-+ features &= ~NETIF_F_GSO_MASK;
-+
-+ /* If encapsulation offload request, verify we are testing
-+ * hardware encapsulation features instead of standard
-+ * features for the netdev
-+ */
-+ if (skb->encapsulation)
-+ features &= dev->hw_enc_features;
-+
-+ if (skb_vlan_tagged(skb))
-+ features = netdev_intersect_features(features,
-+ dev->vlan_features |
-+ NETIF_F_HW_VLAN_CTAG_TX |
-+ NETIF_F_HW_VLAN_STAG_TX);
-+
-+ if (dev->netdev_ops->ndo_features_check)
-+ features &= dev->netdev_ops->ndo_features_check(skb, dev,
-+ features);
-+ else
-+ features &= dflt_features_check(skb, dev, features);
-+
-+ return harmonize_features(skb, features);
-+}
-+EXPORT_SYMBOL(netif_skb_features);
-+
-+static int xmit_one(struct sk_buff *skb, struct net_device *dev,
-+ struct netdev_queue *txq, bool more)
-+{
-+ unsigned int len;
-+ int rc;
-+
-+ if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
-+ dev_queue_xmit_nit(skb, dev);
-+
-+ len = skb->len;
-+ trace_net_dev_start_xmit(skb, dev);
-+ rc = netdev_start_xmit(skb, dev, txq, more);
-+ trace_net_dev_xmit(skb, rc, dev, len);
-+
-+ return rc;
-+}
-+
-+struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
-+ struct netdev_queue *txq, int *ret)
-+{
-+ struct sk_buff *skb = first;
-+ int rc = NETDEV_TX_OK;
-+
-+ while (skb) {
-+ struct sk_buff *next = skb->next;
-+
-+ skb->next = NULL;
-+ rc = xmit_one(skb, dev, txq, next != NULL);
-+ if (unlikely(!dev_xmit_complete(rc))) {
-+ skb->next = next;
-+ goto out;
-+ }
-+
-+ skb = next;
-+ if (netif_xmit_stopped(txq) && skb) {
-+ rc = NETDEV_TX_BUSY;
-+ break;
-+ }
-+ }
-+
-+out:
-+ *ret = rc;
-+ return skb;
-+}
-+
-+static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
-+ netdev_features_t features)
-+{
-+ if (skb_vlan_tag_present(skb) &&
-+ !vlan_hw_offload_capable(features, skb->vlan_proto))
-+ skb = __vlan_hwaccel_push_inside(skb);
-+ return skb;
-+}
-+
-+static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
-+{
-+ netdev_features_t features;
-+
-+ if (skb->next)
-+ return skb;
-+
-+ features = netif_skb_features(skb);
-+ skb = validate_xmit_vlan(skb, features);
-+ if (unlikely(!skb))
-+ goto out_null;
-+
-+ if (netif_needs_gso(skb, features)) {
-+ struct sk_buff *segs;
-+
-+ segs = skb_gso_segment(skb, features);
-+ if (IS_ERR(segs)) {
-+ goto out_kfree_skb;
-+ } else if (segs) {
-+ consume_skb(skb);
-+ skb = segs;
-+ }
-+ } else {
-+ if (skb_needs_linearize(skb, features) &&
-+ __skb_linearize(skb))
-+ goto out_kfree_skb;
-+
-+ /* If packet is not checksummed and device does not
-+ * support checksumming for this protocol, complete
-+ * checksumming here.
-+ */
-+ if (skb->ip_summed == CHECKSUM_PARTIAL) {
-+ if (skb->encapsulation)
-+ skb_set_inner_transport_header(skb,
-+ skb_checksum_start_offset(skb));
-+ else
-+ skb_set_transport_header(skb,
-+ skb_checksum_start_offset(skb));
-+ if (!(features & NETIF_F_ALL_CSUM) &&
-+ skb_checksum_help(skb))
-+ goto out_kfree_skb;
-+ }
-+ }
-+
-+ return skb;
-+
-+out_kfree_skb:
-+ kfree_skb(skb);
-+out_null:
-+ return NULL;
-+}
-+
-+struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
-+{
-+ struct sk_buff *next, *head = NULL, *tail;
-+
-+ for (; skb != NULL; skb = next) {
-+ next = skb->next;
-+ skb->next = NULL;
-+
-+ /* in case skb wont be segmented, point to itself */
-+ skb->prev = skb;
-+
-+ skb = validate_xmit_skb(skb, dev);
-+ if (!skb)
-+ continue;
-+
-+ if (!head)
-+ head = skb;
-+ else
-+ tail->next = skb;
-+ /* If skb was segmented, skb->prev points to
-+ * the last segment. If not, it still contains skb.
-+ */
-+ tail = skb->prev;
-+ }
-+ return head;
-+}
-+
-+static void qdisc_pkt_len_init(struct sk_buff *skb)
-+{
-+ const struct skb_shared_info *shinfo = skb_shinfo(skb);
-+
-+ qdisc_skb_cb(skb)->pkt_len = skb->len;
-+
-+ /* To get more precise estimation of bytes sent on wire,
-+ * we add to pkt_len the headers size of all segments
-+ */
-+ if (shinfo->gso_size) {
-+ unsigned int hdr_len;
-+ u16 gso_segs = shinfo->gso_segs;
-+
-+ /* mac layer + network layer */
-+ hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
-+
-+ /* + transport layer */
-+ if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
-+ hdr_len += tcp_hdrlen(skb);
-+ else
-+ hdr_len += sizeof(struct udphdr);
-+
-+ if (shinfo->gso_type & SKB_GSO_DODGY)
-+ gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
-+ shinfo->gso_size);
-+
-+ qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
-+ }
-+}
-+
-+static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
-+ struct net_device *dev,
-+ struct netdev_queue *txq)
-+{
-+ spinlock_t *root_lock = qdisc_lock(q);
-+ bool contended;
-+ int rc;
-+
-+ qdisc_pkt_len_init(skb);
-+ qdisc_calculate_pkt_len(skb, q);
-+ /*
-+ * Heuristic to force contended enqueues to serialize on a
-+ * separate lock before trying to get qdisc main lock.
-+ * This permits __QDISC___STATE_RUNNING owner to get the lock more
-+ * often and dequeue packets faster.
-+ */
-+ contended = qdisc_is_running(q);
-+ if (unlikely(contended))
-+ spin_lock(&q->busylock);
-+
-+ spin_lock(root_lock);
-+ if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
-+ kfree_skb(skb);
-+ rc = NET_XMIT_DROP;
-+ } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
-+ qdisc_run_begin(q)) {
-+ /*
-+ * This is a work-conserving queue; there are no old skbs
-+ * waiting to be sent out; and the qdisc is not running -
-+ * xmit the skb directly.
-+ */
-+
-+ qdisc_bstats_update(q, skb);
-+
-+ if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
-+ if (unlikely(contended)) {
-+ spin_unlock(&q->busylock);
-+ contended = false;
-+ }
-+ __qdisc_run(q);
-+ } else
-+ qdisc_run_end(q);
-+
-+ rc = NET_XMIT_SUCCESS;
-+ } else {
-+ rc = q->enqueue(skb, q) & NET_XMIT_MASK;
-+ if (qdisc_run_begin(q)) {
-+ if (unlikely(contended)) {
-+ spin_unlock(&q->busylock);
-+ contended = false;
-+ }
-+ __qdisc_run(q);
-+ }
-+ }
-+ spin_unlock(root_lock);
-+ if (unlikely(contended))
-+ spin_unlock(&q->busylock);
-+ return rc;
-+}
-+
-+#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
-+static void skb_update_prio(struct sk_buff *skb)
-+{
-+ struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
-+
-+ if (!skb->priority && skb->sk && map) {
-+ unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
-+
-+ if (prioidx < map->priomap_len)
-+ skb->priority = map->priomap[prioidx];
-+ }
-+}
-+#else
-+#define skb_update_prio(skb)
-+#endif
-+
-+DEFINE_PER_CPU(int, xmit_recursion);
-+EXPORT_SYMBOL(xmit_recursion);
-+
-+#define RECURSION_LIMIT 10
-+
-+/**
-+ * dev_loopback_xmit - loop back @skb
-+ * @skb: buffer to transmit
-+ */
-+int dev_loopback_xmit(struct sock *sk, struct sk_buff *skb)
-+{
-+ skb_reset_mac_header(skb);
-+ __skb_pull(skb, skb_network_offset(skb));
-+ skb->pkt_type = PACKET_LOOPBACK;
-+ skb->ip_summed = CHECKSUM_UNNECESSARY;
-+ WARN_ON(!skb_dst(skb));
-+ skb_dst_force(skb);
-+ netif_rx_ni(skb);
-+ return 0;
-+}
-+EXPORT_SYMBOL(dev_loopback_xmit);
-+
-+/**
-+ * __dev_queue_xmit - transmit a buffer
-+ * @skb: buffer to transmit
-+ * @accel_priv: private data used for L2 forwarding offload
-+ *
-+ * Queue a buffer for transmission to a network device. The caller must
-+ * have set the device and priority and built the buffer before calling
-+ * this function. The function can be called from an interrupt.
-+ *
-+ * A negative errno code is returned on a failure. A success does not
-+ * guarantee the frame will be transmitted as it may be dropped due
-+ * to congestion or traffic shaping.
-+ *
-+ * -----------------------------------------------------------------------------------
-+ * I notice this method can also return errors from the queue disciplines,
-+ * including NET_XMIT_DROP, which is a positive value. So, errors can also
-+ * be positive.
-+ *
-+ * Regardless of the return value, the skb is consumed, so it is currently
-+ * difficult to retry a send to this method. (You can bump the ref count
-+ * before sending to hold a reference for retry if you are careful.)
-+ *
-+ * When calling this method, interrupts MUST be enabled. This is because
-+ * the BH enable code must have IRQs enabled so that it will not deadlock.
-+ * --BLG
-+ */
-+static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
-+{
-+ struct net_device *dev = skb->dev;
-+ struct netdev_queue *txq;
-+ struct Qdisc *q;
-+ int rc = -ENOMEM;
-+
-+ skb_reset_mac_header(skb);
-+
-+ if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
-+ __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
-+
-+ /* Disable soft irqs for various locks below. Also
-+ * stops preemption for RCU.
-+ */
-+ rcu_read_lock_bh();
-+
-+ skb_update_prio(skb);
-+
-+ /* If device/qdisc don't need skb->dst, release it right now while
-+ * its hot in this cpu cache.
-+ */
-+ if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
-+ skb_dst_drop(skb);
-+ else
-+ skb_dst_force(skb);
-+
-+ txq = netdev_pick_tx(dev, skb, accel_priv);
-+ q = rcu_dereference_bh(txq->qdisc);
-+
-+#ifdef CONFIG_NET_CLS_ACT
-+ skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
-+#endif
-+ trace_net_dev_queue(skb);
-+ if (q->enqueue) {
-+ rc = __dev_xmit_skb(skb, q, dev, txq);
-+ goto out;
-+ }
-+
-+ /* The device has no queue. Common case for software devices:
-+ loopback, all the sorts of tunnels...
-+
-+ Really, it is unlikely that netif_tx_lock protection is necessary
-+ here. (f.e. loopback and IP tunnels are clean ignoring statistics
-+ counters.)
-+ However, it is possible, that they rely on protection
-+ made by us here.
-+
-+ Check this and shot the lock. It is not prone from deadlocks.
-+ Either shot noqueue qdisc, it is even simpler 8)
-+ */
-+ if (dev->flags & IFF_UP) {
-+ int cpu = smp_processor_id(); /* ok because BHs are off */
-+
-+ if (txq->xmit_lock_owner != cpu) {
-+
-+ if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
-+ goto recursion_alert;
-+
-+ skb = validate_xmit_skb(skb, dev);
-+ if (!skb)
-+ goto drop;
-+
-+ HARD_TX_LOCK(dev, txq, cpu);
-+
-+ if (!netif_xmit_stopped(txq)) {
-+ __this_cpu_inc(xmit_recursion);
-+ skb = dev_hard_start_xmit(skb, dev, txq, &rc);
-+ __this_cpu_dec(xmit_recursion);
-+ if (dev_xmit_complete(rc)) {
-+ HARD_TX_UNLOCK(dev, txq);
-+ goto out;
-+ }
-+ }
-+ HARD_TX_UNLOCK(dev, txq);
-+ net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
-+ dev->name);
-+ } else {
-+ /* Recursion is detected! It is possible,
-+ * unfortunately
-+ */
-+recursion_alert:
-+ net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
-+ dev->name);
-+ }
-+ }
-+
-+ rc = -ENETDOWN;
-+drop:
-+ rcu_read_unlock_bh();
-+
-+ atomic_long_inc(&dev->tx_dropped);
-+ kfree_skb_list(skb);
-+ return rc;
-+out:
-+ rcu_read_unlock_bh();
-+ return rc;
-+}
-+
-+int dev_queue_xmit_sk(struct sock *sk, struct sk_buff *skb)
-+{
-+ return __dev_queue_xmit(skb, NULL);
-+}
-+EXPORT_SYMBOL(dev_queue_xmit_sk);
-+
-+int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
-+{
-+ return __dev_queue_xmit(skb, accel_priv);
-+}
-+EXPORT_SYMBOL(dev_queue_xmit_accel);
-+
-+
-+/*=======================================================================
-+ Receiver routines
-+ =======================================================================*/
-+
-+int netdev_max_backlog __read_mostly = 1000;
-+EXPORT_SYMBOL(netdev_max_backlog);
-+
-+int netdev_tstamp_prequeue __read_mostly = 1;
-+int netdev_budget __read_mostly = 300;
-+int weight_p __read_mostly = 64; /* old backlog weight */
-+
-+/* Called with irq disabled */
-+static inline void ____napi_schedule(struct softnet_data *sd,
-+ struct napi_struct *napi)
-+{
-+ list_add_tail(&napi->poll_list, &sd->poll_list);
-+ __raise_softirq_irqoff(NET_RX_SOFTIRQ);
-+}
-+
-+#ifdef CONFIG_RPS
-+
-+/* One global table that all flow-based protocols share. */
-+struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
-+EXPORT_SYMBOL(rps_sock_flow_table);
-+u32 rps_cpu_mask __read_mostly;
-+EXPORT_SYMBOL(rps_cpu_mask);
-+
-+struct static_key rps_needed __read_mostly;
-+
-+static struct rps_dev_flow *
-+set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
-+ struct rps_dev_flow *rflow, u16 next_cpu)
-+{
-+ if (next_cpu < nr_cpu_ids) {
-+#ifdef CONFIG_RFS_ACCEL
-+ struct netdev_rx_queue *rxqueue;
-+ struct rps_dev_flow_table *flow_table;
-+ struct rps_dev_flow *old_rflow;
-+ u32 flow_id;
-+ u16 rxq_index;
-+ int rc;
-+
-+ /* Should we steer this flow to a different hardware queue? */
-+ if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
-+ !(dev->features & NETIF_F_NTUPLE))
-+ goto out;
-+ rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
-+ if (rxq_index == skb_get_rx_queue(skb))
-+ goto out;
-+
-+ rxqueue = dev->_rx + rxq_index;
-+ flow_table = rcu_dereference(rxqueue->rps_flow_table);
-+ if (!flow_table)
-+ goto out;
-+ flow_id = skb_get_hash(skb) & flow_table->mask;
-+ rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
-+ rxq_index, flow_id);
-+ if (rc < 0)
-+ goto out;
-+ old_rflow = rflow;
-+ rflow = &flow_table->flows[flow_id];
-+ rflow->filter = rc;
-+ if (old_rflow->filter == rflow->filter)
-+ old_rflow->filter = RPS_NO_FILTER;
-+ out:
-+#endif
-+ rflow->last_qtail =
-+ per_cpu(softnet_data, next_cpu).input_queue_head;
-+ }
-+
-+ rflow->cpu = next_cpu;
-+ return rflow;
-+}
-+
-+/*
-+ * get_rps_cpu is called from netif_receive_skb and returns the target
-+ * CPU from the RPS map of the receiving queue for a given skb.
-+ * rcu_read_lock must be held on entry.
-+ */
-+static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
-+ struct rps_dev_flow **rflowp)
-+{
-+ const struct rps_sock_flow_table *sock_flow_table;
-+ struct netdev_rx_queue *rxqueue = dev->_rx;
-+ struct rps_dev_flow_table *flow_table;
-+ struct rps_map *map;
-+ int cpu = -1;
-+ u32 tcpu;
-+ u32 hash;
-+
-+ if (skb_rx_queue_recorded(skb)) {
-+ u16 index = skb_get_rx_queue(skb);
-+
-+ if (unlikely(index >= dev->real_num_rx_queues)) {
-+ WARN_ONCE(dev->real_num_rx_queues > 1,
-+ "%s received packet on queue %u, but number "
-+ "of RX queues is %u\n",
-+ dev->name, index, dev->real_num_rx_queues);
-+ goto done;
-+ }
-+ rxqueue += index;
-+ }
-+
-+ /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
-+
-+ flow_table = rcu_dereference(rxqueue->rps_flow_table);
-+ map = rcu_dereference(rxqueue->rps_map);
-+ if (!flow_table && !map)
-+ goto done;
-+
-+ skb_reset_network_header(skb);
-+ hash = skb_get_hash(skb);
-+ if (!hash)
-+ goto done;
-+
-+ sock_flow_table = rcu_dereference(rps_sock_flow_table);
-+ if (flow_table && sock_flow_table) {
-+ struct rps_dev_flow *rflow;
-+ u32 next_cpu;
-+ u32 ident;
-+
-+ /* First check into global flow table if there is a match */
-+ ident = sock_flow_table->ents[hash & sock_flow_table->mask];
-+ if ((ident ^ hash) & ~rps_cpu_mask)
-+ goto try_rps;
-+
-+ next_cpu = ident & rps_cpu_mask;
-+
-+ /* OK, now we know there is a match,
-+ * we can look at the local (per receive queue) flow table
-+ */
-+ rflow = &flow_table->flows[hash & flow_table->mask];
-+ tcpu = rflow->cpu;
-+
-+ /*
-+ * If the desired CPU (where last recvmsg was done) is
-+ * different from current CPU (one in the rx-queue flow
-+ * table entry), switch if one of the following holds:
-+ * - Current CPU is unset (>= nr_cpu_ids).
-+ * - Current CPU is offline.
-+ * - The current CPU's queue tail has advanced beyond the
-+ * last packet that was enqueued using this table entry.
-+ * This guarantees that all previous packets for the flow
-+ * have been dequeued, thus preserving in order delivery.
-+ */
-+ if (unlikely(tcpu != next_cpu) &&
-+ (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
-+ ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
-+ rflow->last_qtail)) >= 0)) {
-+ tcpu = next_cpu;
-+ rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
-+ }
-+
-+ if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
-+ *rflowp = rflow;
-+ cpu = tcpu;
-+ goto done;
-+ }
-+ }
-+
-+try_rps:
-+
-+ if (map) {
-+ tcpu = map->cpus[reciprocal_scale(hash, map->len)];
-+ if (cpu_online(tcpu)) {
-+ cpu = tcpu;
-+ goto done;
-+ }
-+ }
-+
-+done:
-+ return cpu;
-+}
-+
-+#ifdef CONFIG_RFS_ACCEL
-+
-+/**
-+ * rps_may_expire_flow - check whether an RFS hardware filter may be removed
-+ * @dev: Device on which the filter was set
-+ * @rxq_index: RX queue index
-+ * @flow_id: Flow ID passed to ndo_rx_flow_steer()
-+ * @filter_id: Filter ID returned by ndo_rx_flow_steer()
-+ *
-+ * Drivers that implement ndo_rx_flow_steer() should periodically call
-+ * this function for each installed filter and remove the filters for
-+ * which it returns %true.
-+ */
-+bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
-+ u32 flow_id, u16 filter_id)
-+{
-+ struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
-+ struct rps_dev_flow_table *flow_table;
-+ struct rps_dev_flow *rflow;
-+ bool expire = true;
-+ unsigned int cpu;
-+
-+ rcu_read_lock();
-+ flow_table = rcu_dereference(rxqueue->rps_flow_table);
-+ if (flow_table && flow_id <= flow_table->mask) {
-+ rflow = &flow_table->flows[flow_id];
-+ cpu = ACCESS_ONCE(rflow->cpu);
-+ if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
-+ ((int)(per_cpu(softnet_data, cpu).input_queue_head -
-+ rflow->last_qtail) <
-+ (int)(10 * flow_table->mask)))
-+ expire = false;
-+ }
-+ rcu_read_unlock();
-+ return expire;
-+}
-+EXPORT_SYMBOL(rps_may_expire_flow);
-+
-+#endif /* CONFIG_RFS_ACCEL */
-+
-+/* Called from hardirq (IPI) context */
-+static void rps_trigger_softirq(void *data)
-+{
-+ struct softnet_data *sd = data;
-+
-+ ____napi_schedule(sd, &sd->backlog);
-+ sd->received_rps++;
-+}
-+
-+#endif /* CONFIG_RPS */
-+
-+/*
-+ * Check if this softnet_data structure is another cpu one
-+ * If yes, queue it to our IPI list and return 1
-+ * If no, return 0
-+ */
-+static int rps_ipi_queued(struct softnet_data *sd)
-+{
-+#ifdef CONFIG_RPS
-+ struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
-+
-+ if (sd != mysd) {
-+ sd->rps_ipi_next = mysd->rps_ipi_list;
-+ mysd->rps_ipi_list = sd;
-+
-+ __raise_softirq_irqoff(NET_RX_SOFTIRQ);
-+ return 1;
-+ }
-+#endif /* CONFIG_RPS */
-+ return 0;
-+}
-+
-+#ifdef CONFIG_NET_FLOW_LIMIT
-+int netdev_flow_limit_table_len __read_mostly = (1 << 12);
-+#endif
-+
-+static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
-+{
-+#ifdef CONFIG_NET_FLOW_LIMIT
-+ struct sd_flow_limit *fl;
-+ struct softnet_data *sd;
-+ unsigned int old_flow, new_flow;
-+
-+ if (qlen < (netdev_max_backlog >> 1))
-+ return false;
-+
-+ sd = this_cpu_ptr(&softnet_data);
-+
-+ rcu_read_lock();
-+ fl = rcu_dereference(sd->flow_limit);
-+ if (fl) {
-+ new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
-+ old_flow = fl->history[fl->history_head];
-+ fl->history[fl->history_head] = new_flow;
-+
-+ fl->history_head++;
-+ fl->history_head &= FLOW_LIMIT_HISTORY - 1;
-+
-+ if (likely(fl->buckets[old_flow]))
-+ fl->buckets[old_flow]--;
-+
-+ if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
-+ fl->count++;
-+ rcu_read_unlock();
-+ return true;
-+ }
-+ }
-+ rcu_read_unlock();
-+#endif
-+ return false;
-+}
-+
-+/*
-+ * enqueue_to_backlog is called to queue an skb to a per CPU backlog
-+ * queue (may be a remote CPU queue).
-+ */
-+static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
-+ unsigned int *qtail)
-+{
-+ struct softnet_data *sd;
-+ unsigned long flags;
-+ unsigned int qlen;
-+
-+ sd = &per_cpu(softnet_data, cpu);
-+
-+ local_irq_save(flags);
-+
-+ rps_lock(sd);
-+ if (!netif_running(skb->dev))
-+ goto drop;
-+ qlen = skb_queue_len(&sd->input_pkt_queue);
-+ if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
-+ if (qlen) {
-+enqueue:
-+ __skb_queue_tail(&sd->input_pkt_queue, skb);
-+ input_queue_tail_incr_save(sd, qtail);
-+ rps_unlock(sd);
-+ local_irq_restore(flags);
-+ return NET_RX_SUCCESS;
-+ }
-+
-+ /* Schedule NAPI for backlog device
-+ * We can use non atomic operation since we own the queue lock
-+ */
-+ if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
-+ if (!rps_ipi_queued(sd))
-+ ____napi_schedule(sd, &sd->backlog);
-+ }
-+ goto enqueue;
-+ }
-+
-+drop:
-+ sd->dropped++;
-+ rps_unlock(sd);
-+
-+ local_irq_restore(flags);
-+
-+ atomic_long_inc(&skb->dev->rx_dropped);
-+ kfree_skb(skb);
-+ return NET_RX_DROP;
-+}
-+
-+static int netif_rx_internal(struct sk_buff *skb)
-+{
-+ int ret;
-+
-+ net_timestamp_check(netdev_tstamp_prequeue, skb);
-+
-+ trace_netif_rx(skb);
-+#ifdef CONFIG_RPS
-+ if (static_key_false(&rps_needed)) {
-+ struct rps_dev_flow voidflow, *rflow = &voidflow;
-+ int cpu;
-+
-+ preempt_disable();
-+ rcu_read_lock();
-+
-+ cpu = get_rps_cpu(skb->dev, skb, &rflow);
-+ if (cpu < 0)
-+ cpu = smp_processor_id();
-+
-+ ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
-+
-+ rcu_read_unlock();
-+ preempt_enable();
-+ } else
-+#endif
-+ {
-+ unsigned int qtail;
-+ ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
-+ put_cpu();
-+ }
-+ return ret;
-+}
-+
-+/**
-+ * netif_rx - post buffer to the network code
-+ * @skb: buffer to post
-+ *
-+ * This function receives a packet from a device driver and queues it for
-+ * the upper (protocol) levels to process. It always succeeds. The buffer
-+ * may be dropped during processing for congestion control or by the
-+ * protocol layers.
-+ *
-+ * return values:
-+ * NET_RX_SUCCESS (no congestion)
-+ * NET_RX_DROP (packet was dropped)
-+ *
-+ */
-+
-+int netif_rx(struct sk_buff *skb)
-+{
-+ trace_netif_rx_entry(skb);
-+
-+ return netif_rx_internal(skb);
-+}
-+EXPORT_SYMBOL(netif_rx);
-+
-+int netif_rx_ni(struct sk_buff *skb)
-+{
-+ int err;
-+
-+ trace_netif_rx_ni_entry(skb);
-+
-+ preempt_disable();
-+ err = netif_rx_internal(skb);
-+ if (local_softirq_pending())
-+ do_softirq();
-+ preempt_enable();
-+
-+ return err;
-+}
-+EXPORT_SYMBOL(netif_rx_ni);
-+
-+static void net_tx_action(struct softirq_action *h)
-+{
-+ struct softnet_data *sd = this_cpu_ptr(&softnet_data);
-+
-+ if (sd->completion_queue) {
-+ struct sk_buff *clist;
-+
-+ local_irq_disable();
-+ clist = sd->completion_queue;
-+ sd->completion_queue = NULL;
-+ local_irq_enable();
-+
-+ while (clist) {
-+ struct sk_buff *skb = clist;
-+ clist = clist->next;
-+
-+ WARN_ON(atomic_read(&skb->users));
-+ if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
-+ trace_consume_skb(skb);
-+ else
-+ trace_kfree_skb(skb, net_tx_action);
-+ __kfree_skb(skb);
-+ }
-+ }
-+
-+ if (sd->output_queue) {
-+ struct Qdisc *head;
-+
-+ local_irq_disable();
-+ head = sd->output_queue;
-+ sd->output_queue = NULL;
-+ sd->output_queue_tailp = &sd->output_queue;
-+ local_irq_enable();
-+
-+ while (head) {
-+ struct Qdisc *q = head;
-+ spinlock_t *root_lock;
-+
-+ head = head->next_sched;
-+
-+ root_lock = qdisc_lock(q);
-+ if (spin_trylock(root_lock)) {
-+ smp_mb__before_atomic();
-+ clear_bit(__QDISC_STATE_SCHED,
-+ &q->state);
-+ qdisc_run(q);
-+ spin_unlock(root_lock);
-+ } else {
-+ if (!test_bit(__QDISC_STATE_DEACTIVATED,
-+ &q->state)) {
-+ __netif_reschedule(q);
-+ } else {
-+ smp_mb__before_atomic();
-+ clear_bit(__QDISC_STATE_SCHED,
-+ &q->state);
-+ }
-+ }
-+ }
-+ }
-+}
-+
-+#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
-+ (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
-+/* This hook is defined here for ATM LANE */
-+int (*br_fdb_test_addr_hook)(struct net_device *dev,
-+ unsigned char *addr) __read_mostly;
-+EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
-+#endif
-+
-+#ifdef CONFIG_NET_CLS_ACT
-+/* TODO: Maybe we should just force sch_ingress to be compiled in
-+ * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
-+ * a compare and 2 stores extra right now if we dont have it on
-+ * but have CONFIG_NET_CLS_ACT
-+ * NOTE: This doesn't stop any functionality; if you dont have
-+ * the ingress scheduler, you just can't add policies on ingress.
-+ *
-+ */
-+static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
-+{
-+ struct net_device *dev = skb->dev;
-+ u32 ttl = G_TC_RTTL(skb->tc_verd);
-+ int result = TC_ACT_OK;
-+ struct Qdisc *q;
-+
-+ if (unlikely(MAX_RED_LOOP < ttl++)) {
-+ net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
-+ skb->skb_iif, dev->ifindex);
-+ return TC_ACT_SHOT;
-+ }
-+
-+ skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
-+ skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
-+
-+ q = rcu_dereference(rxq->qdisc);
-+ if (q != &noop_qdisc) {
-+ spin_lock(qdisc_lock(q));
-+ if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
-+ result = qdisc_enqueue_root(skb, q);
-+ spin_unlock(qdisc_lock(q));
-+ }
-+
-+ return result;
-+}
-+
-+static inline struct sk_buff *handle_ing(struct sk_buff *skb,
-+ struct packet_type **pt_prev,
-+ int *ret, struct net_device *orig_dev)
-+{
-+ struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
-+
-+ if (!rxq || rcu_access_pointer(rxq->qdisc) == &noop_qdisc)
-+ return skb;
-+
-+ if (*pt_prev) {
-+ *ret = deliver_skb(skb, *pt_prev, orig_dev);
-+ *pt_prev = NULL;
-+ }
-+
-+ switch (ing_filter(skb, rxq)) {
-+ case TC_ACT_SHOT:
-+ case TC_ACT_STOLEN:
-+ kfree_skb(skb);
-+ return NULL;
-+ }
-+
-+ return skb;
-+}
-+#endif
-+
-+/**
-+ * netdev_rx_handler_register - register receive handler
-+ * @dev: device to register a handler for
-+ * @rx_handler: receive handler to register
-+ * @rx_handler_data: data pointer that is used by rx handler
-+ *
-+ * Register a receive handler for a device. This handler will then be
-+ * called from __netif_receive_skb. A negative errno code is returned
-+ * on a failure.
-+ *
-+ * The caller must hold the rtnl_mutex.
-+ *
-+ * For a general description of rx_handler, see enum rx_handler_result.
-+ */
-+int netdev_rx_handler_register(struct net_device *dev,
-+ rx_handler_func_t *rx_handler,
-+ void *rx_handler_data)
-+{
-+ ASSERT_RTNL();
-+
-+ if (dev->rx_handler)
-+ return -EBUSY;
-+
-+ /* Note: rx_handler_data must be set before rx_handler */
-+ rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
-+ rcu_assign_pointer(dev->rx_handler, rx_handler);
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
-+
-+/**
-+ * netdev_rx_handler_unregister - unregister receive handler
-+ * @dev: device to unregister a handler from
-+ *
-+ * Unregister a receive handler from a device.
-+ *
-+ * The caller must hold the rtnl_mutex.
-+ */
-+void netdev_rx_handler_unregister(struct net_device *dev)
-+{
-+
-+ ASSERT_RTNL();
-+ RCU_INIT_POINTER(dev->rx_handler, NULL);
-+ /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
-+ * section has a guarantee to see a non NULL rx_handler_data
-+ * as well.
-+ */
-+ synchronize_net();
-+ RCU_INIT_POINTER(dev->rx_handler_data, NULL);
-+}
-+EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
-+
-+/*
-+ * Limit the use of PFMEMALLOC reserves to those protocols that implement
-+ * the special handling of PFMEMALLOC skbs.
-+ */
-+static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
-+{
-+ switch (skb->protocol) {
-+ case htons(ETH_P_ARP):
-+ case htons(ETH_P_IP):
-+ case htons(ETH_P_IPV6):
-+ case htons(ETH_P_8021Q):
-+ case htons(ETH_P_8021AD):
-+ return true;
-+ default:
-+ return false;
-+ }
-+}
-+
-+static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
-+{
-+ struct packet_type *ptype, *pt_prev;
-+ rx_handler_func_t *rx_handler;
-+ struct net_device *orig_dev;
-+ bool deliver_exact = false;
-+ int ret = NET_RX_DROP;
-+ __be16 type;
-+
-+ net_timestamp_check(!netdev_tstamp_prequeue, skb);
-+
-+ trace_netif_receive_skb(skb);
-+
-+ orig_dev = skb->dev;
-+
-+ skb_reset_network_header(skb);
-+ if (!skb_transport_header_was_set(skb))
-+ skb_reset_transport_header(skb);
-+ skb_reset_mac_len(skb);
-+
-+ pt_prev = NULL;
-+
-+another_round:
-+ skb->skb_iif = skb->dev->ifindex;
-+
-+ __this_cpu_inc(softnet_data.processed);
-+
-+ if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
-+ skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
-+ skb = skb_vlan_untag(skb);
-+ if (unlikely(!skb))
-+ goto out;
-+ }
-+
-+#ifdef CONFIG_NET_CLS_ACT
-+ if (skb->tc_verd & TC_NCLS) {
-+ skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
-+ goto ncls;
-+ }
-+#endif
-+
-+ if (pfmemalloc)
-+ goto skip_taps;
-+
-+ list_for_each_entry_rcu(ptype, &ptype_all, list) {
-+ if (pt_prev)
-+ ret = deliver_skb(skb, pt_prev, orig_dev);
-+ pt_prev = ptype;
-+ }
-+
-+ list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
-+ if (pt_prev)
-+ ret = deliver_skb(skb, pt_prev, orig_dev);
-+ pt_prev = ptype;
-+ }
-+
-+skip_taps:
-+#ifdef CONFIG_NET_CLS_ACT
-+ if (static_key_false(&ingress_needed)) {
-+ skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
-+ if (!skb)
-+ goto out;
-+ }
-+
-+ skb->tc_verd = 0;
-+ncls:
-+#endif
-+ if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
-+ goto drop;
-+
-+ if (skb_vlan_tag_present(skb)) {
-+ if (pt_prev) {
-+ ret = deliver_skb(skb, pt_prev, orig_dev);
-+ pt_prev = NULL;
-+ }
-+ if (vlan_do_receive(&skb))
-+ goto another_round;
-+ else if (unlikely(!skb))
-+ goto out;
-+ }
-+
-+ rx_handler = rcu_dereference(skb->dev->rx_handler);
-+ if (rx_handler) {
-+ if (pt_prev) {
-+ ret = deliver_skb(skb, pt_prev, orig_dev);
-+ pt_prev = NULL;
-+ }
-+ switch (rx_handler(&skb)) {
-+ case RX_HANDLER_CONSUMED:
-+ ret = NET_RX_SUCCESS;
-+ goto out;
-+ case RX_HANDLER_ANOTHER:
-+ goto another_round;
-+ case RX_HANDLER_EXACT:
-+ deliver_exact = true;
-+ case RX_HANDLER_PASS:
-+ break;
-+ default:
-+ BUG();
-+ }
-+ }
-+
-+ if (unlikely(skb_vlan_tag_present(skb))) {
-+ if (skb_vlan_tag_get_id(skb))
-+ skb->pkt_type = PACKET_OTHERHOST;
-+ /* Note: we might in the future use prio bits
-+ * and set skb->priority like in vlan_do_receive()
-+ * For the time being, just ignore Priority Code Point
-+ */
-+ skb->vlan_tci = 0;
-+ }
-+
-+ type = skb->protocol;
-+
-+ /* deliver only exact match when indicated */
-+ if (likely(!deliver_exact)) {
-+ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
-+ &ptype_base[ntohs(type) &
-+ PTYPE_HASH_MASK]);
-+ }
-+
-+ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
-+ &orig_dev->ptype_specific);
-+
-+ if (unlikely(skb->dev != orig_dev)) {
-+ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
-+ &skb->dev->ptype_specific);
-+ }
-+
-+ if (pt_prev) {
-+ if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
-+ goto drop;
-+ else
-+ ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
-+ } else {
-+drop:
-+ atomic_long_inc(&skb->dev->rx_dropped);
-+ kfree_skb(skb);
-+ /* Jamal, now you will not able to escape explaining
-+ * me how you were going to use this. :-)
-+ */
-+ ret = NET_RX_DROP;
-+ }
-+
-+out:
-+ return ret;
-+}
-+
-+static int __netif_receive_skb(struct sk_buff *skb)
-+{
-+ int ret;
-+
-+ if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
-+ unsigned long pflags = current->flags;
-+
-+ /*
-+ * PFMEMALLOC skbs are special, they should
-+ * - be delivered to SOCK_MEMALLOC sockets only
-+ * - stay away from userspace
-+ * - have bounded memory usage
-+ *
-+ * Use PF_MEMALLOC as this saves us from propagating the allocation
-+ * context down to all allocation sites.
-+ */
-+ current->flags |= PF_MEMALLOC;
-+ ret = __netif_receive_skb_core(skb, true);
-+ tsk_restore_flags(current, pflags, PF_MEMALLOC);
-+ } else
-+ ret = __netif_receive_skb_core(skb, false);
-+
-+ return ret;
-+}
-+
-+static int netif_receive_skb_internal(struct sk_buff *skb)
-+{
-+ int ret;
-+
-+ net_timestamp_check(netdev_tstamp_prequeue, skb);
-+
-+ if (skb_defer_rx_timestamp(skb))
-+ return NET_RX_SUCCESS;
-+
-+ rcu_read_lock();
-+
-+#ifdef CONFIG_RPS
-+ if (static_key_false(&rps_needed)) {
-+ struct rps_dev_flow voidflow, *rflow = &voidflow;
-+ int cpu = get_rps_cpu(skb->dev, skb, &rflow);
-+
-+ if (cpu >= 0) {
-+ ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
-+ rcu_read_unlock();
-+ return ret;
-+ }
-+ }
-+#endif
-+ ret = __netif_receive_skb(skb);
-+ rcu_read_unlock();
-+ return ret;
-+}
-+
-+/**
-+ * netif_receive_skb - process receive buffer from network
-+ * @skb: buffer to process
-+ *
-+ * netif_receive_skb() is the main receive data processing function.
-+ * It always succeeds. The buffer may be dropped during processing
-+ * for congestion control or by the protocol layers.
-+ *
-+ * This function may only be called from softirq context and interrupts
-+ * should be enabled.
-+ *
-+ * Return values (usually ignored):
-+ * NET_RX_SUCCESS: no congestion
-+ * NET_RX_DROP: packet was dropped
-+ */
-+int netif_receive_skb_sk(struct sock *sk, struct sk_buff *skb)
-+{
-+ trace_netif_receive_skb_entry(skb);
-+
-+ return netif_receive_skb_internal(skb);
-+}
-+EXPORT_SYMBOL(netif_receive_skb_sk);
-+
-+/* Network device is going away, flush any packets still pending
-+ * Called with irqs disabled.
-+ */
-+static void flush_backlog(void *arg)
-+{
-+ struct net_device *dev = arg;
-+ struct softnet_data *sd = this_cpu_ptr(&softnet_data);
-+ struct sk_buff *skb, *tmp;
-+
-+ rps_lock(sd);
-+ skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
-+ if (skb->dev == dev) {
-+ __skb_unlink(skb, &sd->input_pkt_queue);
-+ kfree_skb(skb);
-+ input_queue_head_incr(sd);
-+ }
-+ }
-+ rps_unlock(sd);
-+
-+ skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
-+ if (skb->dev == dev) {
-+ __skb_unlink(skb, &sd->process_queue);
-+ kfree_skb(skb);
-+ input_queue_head_incr(sd);
-+ }
-+ }
-+}
-+
-+static int napi_gro_complete(struct sk_buff *skb)
-+{
-+ struct packet_offload *ptype;
-+ __be16 type = skb->protocol;
-+ struct list_head *head = &offload_base;
-+ int err = -ENOENT;
-+
-+ BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
-+
-+ if (NAPI_GRO_CB(skb)->count == 1) {
-+ skb_shinfo(skb)->gso_size = 0;
-+ goto out;
-+ }
-+
-+ rcu_read_lock();
-+ list_for_each_entry_rcu(ptype, head, list) {
-+ if (ptype->type != type || !ptype->callbacks.gro_complete)
-+ continue;
-+
-+ err = ptype->callbacks.gro_complete(skb, 0);
-+ break;
-+ }
-+ rcu_read_unlock();
-+
-+ if (err) {
-+ WARN_ON(&ptype->list == head);
-+ kfree_skb(skb);
-+ return NET_RX_SUCCESS;
-+ }
-+
-+out:
-+ return netif_receive_skb_internal(skb);
-+}
-+
-+/* napi->gro_list contains packets ordered by age.
-+ * youngest packets at the head of it.
-+ * Complete skbs in reverse order to reduce latencies.
-+ */
-+void napi_gro_flush(struct napi_struct *napi, bool flush_old)
-+{
-+ struct sk_buff *skb, *prev = NULL;
-+
-+ /* scan list and build reverse chain */
-+ for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
-+ skb->prev = prev;
-+ prev = skb;
-+ }
-+
-+ for (skb = prev; skb; skb = prev) {
-+ skb->next = NULL;
-+
-+ if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
-+ return;
-+
-+ prev = skb->prev;
-+ napi_gro_complete(skb);
-+ napi->gro_count--;
-+ }
-+
-+ napi->gro_list = NULL;
-+}
-+EXPORT_SYMBOL(napi_gro_flush);
-+
-+static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
-+{
-+ struct sk_buff *p;
-+ unsigned int maclen = skb->dev->hard_header_len;
-+ u32 hash = skb_get_hash_raw(skb);
-+
-+ for (p = napi->gro_list; p; p = p->next) {
-+ unsigned long diffs;
-+
-+ NAPI_GRO_CB(p)->flush = 0;
-+
-+ if (hash != skb_get_hash_raw(p)) {
-+ NAPI_GRO_CB(p)->same_flow = 0;
-+ continue;
-+ }
-+
-+ diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
-+ diffs |= p->vlan_tci ^ skb->vlan_tci;
-+ if (maclen == ETH_HLEN)
-+ diffs |= compare_ether_header(skb_mac_header(p),
-+ skb_mac_header(skb));
-+ else if (!diffs)
-+ diffs = memcmp(skb_mac_header(p),
-+ skb_mac_header(skb),
-+ maclen);
-+ NAPI_GRO_CB(p)->same_flow = !diffs;
-+ }
-+}
-+
-+static void skb_gro_reset_offset(struct sk_buff *skb)
-+{
-+ const struct skb_shared_info *pinfo = skb_shinfo(skb);
-+ const skb_frag_t *frag0 = &pinfo->frags[0];
-+
-+ NAPI_GRO_CB(skb)->data_offset = 0;
-+ NAPI_GRO_CB(skb)->frag0 = NULL;
-+ NAPI_GRO_CB(skb)->frag0_len = 0;
-+
-+ if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
-+ pinfo->nr_frags &&
-+ !PageHighMem(skb_frag_page(frag0))) {
-+ NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
-+ NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
-+ }
-+}
-+
-+static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
-+{
-+ struct skb_shared_info *pinfo = skb_shinfo(skb);
-+
-+ BUG_ON(skb->end - skb->tail < grow);
-+
-+ memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
-+
-+ skb->data_len -= grow;
-+ skb->tail += grow;
-+
-+ pinfo->frags[0].page_offset += grow;
-+ skb_frag_size_sub(&pinfo->frags[0], grow);
-+
-+ if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
-+ skb_frag_unref(skb, 0);
-+ memmove(pinfo->frags, pinfo->frags + 1,
-+ --pinfo->nr_frags * sizeof(pinfo->frags[0]));
-+ }
-+}
-+
-+static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
-+{
-+ struct sk_buff **pp = NULL;
-+ struct packet_offload *ptype;
-+ __be16 type = skb->protocol;
-+ struct list_head *head = &offload_base;
-+ int same_flow;
-+ enum gro_result ret;
-+ int grow;
-+
-+ if (!(skb->dev->features & NETIF_F_GRO))
-+ goto normal;
-+
-+ if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
-+ goto normal;
-+
-+ gro_list_prepare(napi, skb);
-+
-+ rcu_read_lock();
-+ list_for_each_entry_rcu(ptype, head, list) {
-+ if (ptype->type != type || !ptype->callbacks.gro_receive)
-+ continue;
-+
-+ skb_set_network_header(skb, skb_gro_offset(skb));
-+ skb_reset_mac_len(skb);
-+ NAPI_GRO_CB(skb)->same_flow = 0;
-+ NAPI_GRO_CB(skb)->flush = 0;
-+ NAPI_GRO_CB(skb)->free = 0;
-+ NAPI_GRO_CB(skb)->udp_mark = 0;
-+ NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
-+
-+ /* Setup for GRO checksum validation */
-+ switch (skb->ip_summed) {
-+ case CHECKSUM_COMPLETE:
-+ NAPI_GRO_CB(skb)->csum = skb->csum;
-+ NAPI_GRO_CB(skb)->csum_valid = 1;
-+ NAPI_GRO_CB(skb)->csum_cnt = 0;
-+ break;
-+ case CHECKSUM_UNNECESSARY:
-+ NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
-+ NAPI_GRO_CB(skb)->csum_valid = 0;
-+ break;
-+ default:
-+ NAPI_GRO_CB(skb)->csum_cnt = 0;
-+ NAPI_GRO_CB(skb)->csum_valid = 0;
-+ }
-+
-+ pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
-+ break;
-+ }
-+ rcu_read_unlock();
-+
-+ if (&ptype->list == head)
-+ goto normal;
-+
-+ same_flow = NAPI_GRO_CB(skb)->same_flow;
-+ ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
-+
-+ if (pp) {
-+ struct sk_buff *nskb = *pp;
-+
-+ *pp = nskb->next;
-+ nskb->next = NULL;
-+ napi_gro_complete(nskb);
-+ napi->gro_count--;
-+ }
-+
-+ if (same_flow)
-+ goto ok;
-+
-+ if (NAPI_GRO_CB(skb)->flush)
-+ goto normal;
-+
-+ if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
-+ struct sk_buff *nskb = napi->gro_list;
-+
-+ /* locate the end of the list to select the 'oldest' flow */
-+ while (nskb->next) {
-+ pp = &nskb->next;
-+ nskb = *pp;
-+ }
-+ *pp = NULL;
-+ nskb->next = NULL;
-+ napi_gro_complete(nskb);
-+ } else {
-+ napi->gro_count++;
-+ }
-+ NAPI_GRO_CB(skb)->count = 1;
-+ NAPI_GRO_CB(skb)->age = jiffies;
-+ NAPI_GRO_CB(skb)->last = skb;
-+ skb_shinfo(skb)->gso_size = skb_gro_len(skb);
-+ skb->next = napi->gro_list;
-+ napi->gro_list = skb;
-+ ret = GRO_HELD;
-+
-+pull:
-+ grow = skb_gro_offset(skb) - skb_headlen(skb);
-+ if (grow > 0)
-+ gro_pull_from_frag0(skb, grow);
-+ok:
-+ return ret;
-+
-+normal:
-+ ret = GRO_NORMAL;
-+ goto pull;
-+}
-+
-+struct packet_offload *gro_find_receive_by_type(__be16 type)
-+{
-+ struct list_head *offload_head = &offload_base;
-+ struct packet_offload *ptype;
-+
-+ list_for_each_entry_rcu(ptype, offload_head, list) {
-+ if (ptype->type != type || !ptype->callbacks.gro_receive)
-+ continue;
-+ return ptype;
-+ }
-+ return NULL;
-+}
-+EXPORT_SYMBOL(gro_find_receive_by_type);
-+
-+struct packet_offload *gro_find_complete_by_type(__be16 type)
-+{
-+ struct list_head *offload_head = &offload_base;
-+ struct packet_offload *ptype;
-+
-+ list_for_each_entry_rcu(ptype, offload_head, list) {
-+ if (ptype->type != type || !ptype->callbacks.gro_complete)
-+ continue;
-+ return ptype;
-+ }
-+ return NULL;
-+}
-+EXPORT_SYMBOL(gro_find_complete_by_type);
-+
-+static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
-+{
-+ switch (ret) {
-+ case GRO_NORMAL:
-+ if (netif_receive_skb_internal(skb))
-+ ret = GRO_DROP;
-+ break;
-+
-+ case GRO_DROP:
-+ kfree_skb(skb);
-+ break;
-+
-+ case GRO_MERGED_FREE:
-+ if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
-+ kmem_cache_free(skbuff_head_cache, skb);
-+ else
-+ __kfree_skb(skb);
-+ break;
-+
-+ case GRO_HELD:
-+ case GRO_MERGED:
-+ break;
-+ }
-+
-+ return ret;
-+}
-+
-+gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
-+{
-+ trace_napi_gro_receive_entry(skb);
-+
-+ skb_gro_reset_offset(skb);
-+
-+ return napi_skb_finish(dev_gro_receive(napi, skb), skb);
-+}
-+EXPORT_SYMBOL(napi_gro_receive);
-+
-+static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
-+{
-+ if (unlikely(skb->pfmemalloc)) {
-+ consume_skb(skb);
-+ return;
-+ }
-+ __skb_pull(skb, skb_headlen(skb));
-+ /* restore the reserve we had after netdev_alloc_skb_ip_align() */
-+ skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
-+ skb->vlan_tci = 0;
-+ skb->dev = napi->dev;
-+ skb->skb_iif = 0;
-+ skb->encapsulation = 0;
-+ skb_shinfo(skb)->gso_type = 0;
-+ skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
-+
-+ napi->skb = skb;
-+}
-+
-+struct sk_buff *napi_get_frags(struct napi_struct *napi)
-+{
-+ struct sk_buff *skb = napi->skb;
-+
-+ if (!skb) {
-+ skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
-+ napi->skb = skb;
-+ }
-+ return skb;
-+}
-+EXPORT_SYMBOL(napi_get_frags);
-+
-+static gro_result_t napi_frags_finish(struct napi_struct *napi,
-+ struct sk_buff *skb,
-+ gro_result_t ret)
-+{
-+ switch (ret) {
-+ case GRO_NORMAL:
-+ case GRO_HELD:
-+ __skb_push(skb, ETH_HLEN);
-+ skb->protocol = eth_type_trans(skb, skb->dev);
-+ if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
-+ ret = GRO_DROP;
-+ break;
-+
-+ case GRO_DROP:
-+ case GRO_MERGED_FREE:
-+ napi_reuse_skb(napi, skb);
-+ break;
-+
-+ case GRO_MERGED:
-+ break;
-+ }
-+
-+ return ret;
-+}
-+
-+/* Upper GRO stack assumes network header starts at gro_offset=0
-+ * Drivers could call both napi_gro_frags() and napi_gro_receive()
-+ * We copy ethernet header into skb->data to have a common layout.
-+ */
-+static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
-+{
-+ struct sk_buff *skb = napi->skb;
-+ const struct ethhdr *eth;
-+ unsigned int hlen = sizeof(*eth);
-+
-+ napi->skb = NULL;
-+
-+ skb_reset_mac_header(skb);
-+ skb_gro_reset_offset(skb);
-+
-+ eth = skb_gro_header_fast(skb, 0);
-+ if (unlikely(skb_gro_header_hard(skb, hlen))) {
-+ eth = skb_gro_header_slow(skb, hlen, 0);
-+ if (unlikely(!eth)) {
-+ napi_reuse_skb(napi, skb);
-+ return NULL;
-+ }
-+ } else {
-+ gro_pull_from_frag0(skb, hlen);
-+ NAPI_GRO_CB(skb)->frag0 += hlen;
-+ NAPI_GRO_CB(skb)->frag0_len -= hlen;
-+ }
-+ __skb_pull(skb, hlen);
-+
-+ /*
-+ * This works because the only protocols we care about don't require
-+ * special handling.
-+ * We'll fix it up properly in napi_frags_finish()
-+ */
-+ skb->protocol = eth->h_proto;
-+
-+ return skb;
-+}
-+
-+gro_result_t napi_gro_frags(struct napi_struct *napi)
-+{
-+ struct sk_buff *skb = napi_frags_skb(napi);
-+
-+ if (!skb)
-+ return GRO_DROP;
-+
-+ trace_napi_gro_frags_entry(skb);
-+
-+ return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
-+}
-+EXPORT_SYMBOL(napi_gro_frags);
-+
-+/* Compute the checksum from gro_offset and return the folded value
-+ * after adding in any pseudo checksum.
-+ */
-+__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
-+{
-+ __wsum wsum;
-+ __sum16 sum;
-+
-+ wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
-+
-+ /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
-+ sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
-+ if (likely(!sum)) {
-+ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
-+ !skb->csum_complete_sw)
-+ netdev_rx_csum_fault(skb->dev);
-+ }
-+
-+ NAPI_GRO_CB(skb)->csum = wsum;
-+ NAPI_GRO_CB(skb)->csum_valid = 1;
-+
-+ return sum;
-+}
-+EXPORT_SYMBOL(__skb_gro_checksum_complete);
-+
-+/*
-+ * net_rps_action_and_irq_enable sends any pending IPI's for rps.
-+ * Note: called with local irq disabled, but exits with local irq enabled.
-+ */
-+static void net_rps_action_and_irq_enable(struct softnet_data *sd)
-+{
-+#ifdef CONFIG_RPS
-+ struct softnet_data *remsd = sd->rps_ipi_list;
-+
-+ if (remsd) {
-+ sd->rps_ipi_list = NULL;
-+
-+ local_irq_enable();
-+
-+ /* Send pending IPI's to kick RPS processing on remote cpus. */
-+ while (remsd) {
-+ struct softnet_data *next = remsd->rps_ipi_next;
-+
-+ if (cpu_online(remsd->cpu))
-+ smp_call_function_single_async(remsd->cpu,
-+ &remsd->csd);
-+ remsd = next;
-+ }
-+ } else
-+#endif
-+ local_irq_enable();
-+}
-+
-+static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
-+{
-+#ifdef CONFIG_RPS
-+ return sd->rps_ipi_list != NULL;
-+#else
-+ return false;
-+#endif
-+}
-+
-+static int process_backlog(struct napi_struct *napi, int quota)
-+{
-+ int work = 0;
-+ struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
-+
-+ /* Check if we have pending ipi, its better to send them now,
-+ * not waiting net_rx_action() end.
-+ */
-+ if (sd_has_rps_ipi_waiting(sd)) {
-+ local_irq_disable();
-+ net_rps_action_and_irq_enable(sd);
-+ }
-+
-+ napi->weight = weight_p;
-+ local_irq_disable();
-+ while (1) {
-+ struct sk_buff *skb;
-+
-+ while ((skb = __skb_dequeue(&sd->process_queue))) {
-+ rcu_read_lock();
-+ local_irq_enable();
-+ __netif_receive_skb(skb);
-+ rcu_read_unlock();
-+ local_irq_disable();
-+ input_queue_head_incr(sd);
-+ if (++work >= quota) {
-+ local_irq_enable();
-+ return work;
-+ }
-+ }
-+
-+ rps_lock(sd);
-+ if (skb_queue_empty(&sd->input_pkt_queue)) {
-+ /*
-+ * Inline a custom version of __napi_complete().
-+ * only current cpu owns and manipulates this napi,
-+ * and NAPI_STATE_SCHED is the only possible flag set
-+ * on backlog.
-+ * We can use a plain write instead of clear_bit(),
-+ * and we dont need an smp_mb() memory barrier.
-+ */
-+ napi->state = 0;
-+ rps_unlock(sd);
-+
-+ break;
-+ }
-+
-+ skb_queue_splice_tail_init(&sd->input_pkt_queue,
-+ &sd->process_queue);
-+ rps_unlock(sd);
-+ }
-+ local_irq_enable();
-+
-+ return work;
-+}
-+
-+/**
-+ * __napi_schedule - schedule for receive
-+ * @n: entry to schedule
-+ *
-+ * The entry's receive function will be scheduled to run.
-+ * Consider using __napi_schedule_irqoff() if hard irqs are masked.
-+ */
-+void __napi_schedule(struct napi_struct *n)
-+{
-+ unsigned long flags;
-+
-+ local_irq_save(flags);
-+ ____napi_schedule(this_cpu_ptr(&softnet_data), n);
-+ local_irq_restore(flags);
-+}
-+EXPORT_SYMBOL(__napi_schedule);
-+
-+/**
-+ * __napi_schedule_irqoff - schedule for receive
-+ * @n: entry to schedule
-+ *
-+ * Variant of __napi_schedule() assuming hard irqs are masked
-+ */
-+void __napi_schedule_irqoff(struct napi_struct *n)
-+{
-+ ____napi_schedule(this_cpu_ptr(&softnet_data), n);
-+}
-+EXPORT_SYMBOL(__napi_schedule_irqoff);
-+
-+void __napi_complete(struct napi_struct *n)
-+{
-+ BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
-+
-+ list_del_init(&n->poll_list);
-+ smp_mb__before_atomic();
-+ clear_bit(NAPI_STATE_SCHED, &n->state);
-+}
-+EXPORT_SYMBOL(__napi_complete);
-+
-+void napi_complete_done(struct napi_struct *n, int work_done)
-+{
-+ unsigned long flags;
-+
-+ /*
-+ * don't let napi dequeue from the cpu poll list
-+ * just in case its running on a different cpu
-+ */
-+ if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
-+ return;
-+
-+ if (n->gro_list) {
-+ unsigned long timeout = 0;
-+
-+ if (work_done)
-+ timeout = n->dev->gro_flush_timeout;
-+
-+ if (timeout)
-+ hrtimer_start(&n->timer, ns_to_ktime(timeout),
-+ HRTIMER_MODE_REL_PINNED);
-+ else
-+ napi_gro_flush(n, false);
-+ }
-+ if (likely(list_empty(&n->poll_list))) {
-+ WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
-+ } else {
-+ /* If n->poll_list is not empty, we need to mask irqs */
-+ local_irq_save(flags);
-+ __napi_complete(n);
-+ local_irq_restore(flags);
-+ }
-+}
-+EXPORT_SYMBOL(napi_complete_done);
-+
-+/* must be called under rcu_read_lock(), as we dont take a reference */
-+struct napi_struct *napi_by_id(unsigned int napi_id)
-+{
-+ unsigned int hash = napi_id % HASH_SIZE(napi_hash);
-+ struct napi_struct *napi;
-+
-+ hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
-+ if (napi->napi_id == napi_id)
-+ return napi;
-+
-+ return NULL;
-+}
-+EXPORT_SYMBOL_GPL(napi_by_id);
-+
-+void napi_hash_add(struct napi_struct *napi)
-+{
-+ if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) {
-+
-+ spin_lock(&napi_hash_lock);
-+
-+ /* 0 is not a valid id, we also skip an id that is taken
-+ * we expect both events to be extremely rare
-+ */
-+ napi->napi_id = 0;
-+ while (!napi->napi_id) {
-+ napi->napi_id = ++napi_gen_id;
-+ if (napi_by_id(napi->napi_id))
-+ napi->napi_id = 0;
-+ }
-+
-+ hlist_add_head_rcu(&napi->napi_hash_node,
-+ &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
-+
-+ spin_unlock(&napi_hash_lock);
-+ }
-+}
-+EXPORT_SYMBOL_GPL(napi_hash_add);
-+
-+/* Warning : caller is responsible to make sure rcu grace period
-+ * is respected before freeing memory containing @napi
-+ */
-+void napi_hash_del(struct napi_struct *napi)
-+{
-+ spin_lock(&napi_hash_lock);
-+
-+ if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
-+ hlist_del_rcu(&napi->napi_hash_node);
-+
-+ spin_unlock(&napi_hash_lock);
-+}
-+EXPORT_SYMBOL_GPL(napi_hash_del);
-+
-+static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
-+{
-+ struct napi_struct *napi;
-+
-+ napi = container_of(timer, struct napi_struct, timer);
-+ if (napi->gro_list)
-+ napi_schedule(napi);
-+
-+ return HRTIMER_NORESTART;
-+}
-+
-+void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
-+ int (*poll)(struct napi_struct *, int), int weight)
-+{
-+ INIT_LIST_HEAD(&napi->poll_list);
-+ hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
-+ napi->timer.function = napi_watchdog;
-+ napi->gro_count = 0;
-+ napi->gro_list = NULL;
-+ napi->skb = NULL;
-+ napi->poll = poll;
-+ if (weight > NAPI_POLL_WEIGHT)
-+ pr_err_once("netif_napi_add() called with weight %d on device %s\n",
-+ weight, dev->name);
-+ napi->weight = weight;
-+ list_add(&napi->dev_list, &dev->napi_list);
-+ napi->dev = dev;
-+#ifdef CONFIG_NETPOLL
-+ spin_lock_init(&napi->poll_lock);
-+ napi->poll_owner = -1;
-+#endif
-+ set_bit(NAPI_STATE_SCHED, &napi->state);
-+}
-+EXPORT_SYMBOL(netif_napi_add);
-+
-+void napi_disable(struct napi_struct *n)
-+{
-+ might_sleep();
-+ set_bit(NAPI_STATE_DISABLE, &n->state);
-+
-+ while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
-+ msleep(1);
-+
-+ hrtimer_cancel(&n->timer);
-+
-+ clear_bit(NAPI_STATE_DISABLE, &n->state);
-+}
-+EXPORT_SYMBOL(napi_disable);
-+
-+void netif_napi_del(struct napi_struct *napi)
-+{
-+ list_del_init(&napi->dev_list);
-+ napi_free_frags(napi);
-+
-+ kfree_skb_list(napi->gro_list);
-+ napi->gro_list = NULL;
-+ napi->gro_count = 0;
-+}
-+EXPORT_SYMBOL(netif_napi_del);
-+
-+static int napi_poll(struct napi_struct *n, struct list_head *repoll)
-+{
-+ void *have;
-+ int work, weight;
-+
-+ list_del_init(&n->poll_list);
-+
-+ have = netpoll_poll_lock(n);
-+
-+ weight = n->weight;
-+
-+ /* This NAPI_STATE_SCHED test is for avoiding a race
-+ * with netpoll's poll_napi(). Only the entity which
-+ * obtains the lock and sees NAPI_STATE_SCHED set will
-+ * actually make the ->poll() call. Therefore we avoid
-+ * accidentally calling ->poll() when NAPI is not scheduled.
-+ */
-+ work = 0;
-+ if (test_bit(NAPI_STATE_SCHED, &n->state)) {
-+ work = n->poll(n, weight);
-+ trace_napi_poll(n);
-+ }
-+
-+ WARN_ON_ONCE(work > weight);
-+
-+ if (likely(work < weight))
-+ goto out_unlock;
-+
-+ /* Drivers must not modify the NAPI state if they
-+ * consume the entire weight. In such cases this code
-+ * still "owns" the NAPI instance and therefore can
-+ * move the instance around on the list at-will.
-+ */
-+ if (unlikely(napi_disable_pending(n))) {
-+ napi_complete(n);
-+ goto out_unlock;
-+ }
-+
-+ if (n->gro_list) {
-+ /* flush too old packets
-+ * If HZ < 1000, flush all packets.
-+ */
-+ napi_gro_flush(n, HZ >= 1000);
-+ }
-+
-+ /* Some drivers may have called napi_schedule
-+ * prior to exhausting their budget.
-+ */
-+ if (unlikely(!list_empty(&n->poll_list))) {
-+ pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
-+ n->dev ? n->dev->name : "backlog");
-+ goto out_unlock;
-+ }
-+
-+ list_add_tail(&n->poll_list, repoll);
-+
-+out_unlock:
-+ netpoll_poll_unlock(have);
-+
-+ return work;
-+}
-+
-+static void net_rx_action(struct softirq_action *h)
-+{
-+ struct softnet_data *sd = this_cpu_ptr(&softnet_data);
-+ unsigned long time_limit = jiffies + 2;
-+ int budget = netdev_budget;
-+ LIST_HEAD(list);
-+ LIST_HEAD(repoll);
-+
-+ local_irq_disable();
-+ list_splice_init(&sd->poll_list, &list);
-+ local_irq_enable();
-+
-+ for (;;) {
-+ struct napi_struct *n;
-+
-+ if (list_empty(&list)) {
-+ if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
-+ return;
-+ break;
-+ }
-+
-+ n = list_first_entry(&list, struct napi_struct, poll_list);
-+ budget -= napi_poll(n, &repoll);
-+
-+ /* If softirq window is exhausted then punt.
-+ * Allow this to run for 2 jiffies since which will allow
-+ * an average latency of 1.5/HZ.
-+ */
-+ if (unlikely(budget <= 0 ||
-+ time_after_eq(jiffies, time_limit))) {
-+ sd->time_squeeze++;
-+ break;
-+ }
-+ }
-+
-+ local_irq_disable();
-+
-+ list_splice_tail_init(&sd->poll_list, &list);
-+ list_splice_tail(&repoll, &list);
-+ list_splice(&list, &sd->poll_list);
-+ if (!list_empty(&sd->poll_list))
-+ __raise_softirq_irqoff(NET_RX_SOFTIRQ);
-+
-+ net_rps_action_and_irq_enable(sd);
-+}
-+
-+struct netdev_adjacent {
-+ struct net_device *dev;
-+
-+ /* upper master flag, there can only be one master device per list */
-+ bool master;
-+
-+ /* counter for the number of times this device was added to us */
-+ u16 ref_nr;
-+
-+ /* private field for the users */
-+ void *private;
-+
-+ struct list_head list;
-+ struct rcu_head rcu;
-+};
-+
-+static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev,
-+ struct net_device *adj_dev,
-+ struct list_head *adj_list)
-+{
-+ struct netdev_adjacent *adj;
-+
-+ list_for_each_entry(adj, adj_list, list) {
-+ if (adj->dev == adj_dev)
-+ return adj;
-+ }
-+ return NULL;
-+}
-+
-+/**
-+ * netdev_has_upper_dev - Check if device is linked to an upper device
-+ * @dev: device
-+ * @upper_dev: upper device to check
-+ *
-+ * Find out if a device is linked to specified upper device and return true
-+ * in case it is. Note that this checks only immediate upper device,
-+ * not through a complete stack of devices. The caller must hold the RTNL lock.
-+ */
-+bool netdev_has_upper_dev(struct net_device *dev,
-+ struct net_device *upper_dev)
-+{
-+ ASSERT_RTNL();
-+
-+ return __netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper);
-+}
-+EXPORT_SYMBOL(netdev_has_upper_dev);
-+
-+/**
-+ * netdev_has_any_upper_dev - Check if device is linked to some device
-+ * @dev: device
-+ *
-+ * Find out if a device is linked to an upper device and return true in case
-+ * it is. The caller must hold the RTNL lock.
-+ */
-+static bool netdev_has_any_upper_dev(struct net_device *dev)
-+{
-+ ASSERT_RTNL();
-+
-+ return !list_empty(&dev->all_adj_list.upper);
-+}
-+
-+/**
-+ * netdev_master_upper_dev_get - Get master upper device
-+ * @dev: device
-+ *
-+ * Find a master upper device and return pointer to it or NULL in case
-+ * it's not there. The caller must hold the RTNL lock.
-+ */
-+struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
-+{
-+ struct netdev_adjacent *upper;
-+
-+ ASSERT_RTNL();
-+
-+ if (list_empty(&dev->adj_list.upper))
-+ return NULL;
-+
-+ upper = list_first_entry(&dev->adj_list.upper,
-+ struct netdev_adjacent, list);
-+ if (likely(upper->master))
-+ return upper->dev;
-+ return NULL;
-+}
-+EXPORT_SYMBOL(netdev_master_upper_dev_get);
-+
-+void *netdev_adjacent_get_private(struct list_head *adj_list)
-+{
-+ struct netdev_adjacent *adj;
-+
-+ adj = list_entry(adj_list, struct netdev_adjacent, list);
-+
-+ return adj->private;
-+}
-+EXPORT_SYMBOL(netdev_adjacent_get_private);
-+
-+/**
-+ * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
-+ * @dev: device
-+ * @iter: list_head ** of the current position
-+ *
-+ * Gets the next device from the dev's upper list, starting from iter
-+ * position. The caller must hold RCU read lock.
-+ */
-+struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
-+ struct list_head **iter)
-+{
-+ struct netdev_adjacent *upper;
-+
-+ WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
-+
-+ upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
-+
-+ if (&upper->list == &dev->adj_list.upper)
-+ return NULL;
-+
-+ *iter = &upper->list;
-+
-+ return upper->dev;
-+}
-+EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
-+
-+/**
-+ * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
-+ * @dev: device
-+ * @iter: list_head ** of the current position
-+ *
-+ * Gets the next device from the dev's upper list, starting from iter
-+ * position. The caller must hold RCU read lock.
-+ */
-+struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
-+ struct list_head **iter)
-+{
-+ struct netdev_adjacent *upper;
-+
-+ WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
-+
-+ upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
-+
-+ if (&upper->list == &dev->all_adj_list.upper)
-+ return NULL;
-+
-+ *iter = &upper->list;
-+
-+ return upper->dev;
-+}
-+EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
-+
-+/**
-+ * netdev_lower_get_next_private - Get the next ->private from the
-+ * lower neighbour list
-+ * @dev: device
-+ * @iter: list_head ** of the current position
-+ *
-+ * Gets the next netdev_adjacent->private from the dev's lower neighbour
-+ * list, starting from iter position. The caller must hold either hold the
-+ * RTNL lock or its own locking that guarantees that the neighbour lower
-+ * list will remain unchainged.
-+ */
-+void *netdev_lower_get_next_private(struct net_device *dev,
-+ struct list_head **iter)
-+{
-+ struct netdev_adjacent *lower;
-+
-+ lower = list_entry(*iter, struct netdev_adjacent, list);
-+
-+ if (&lower->list == &dev->adj_list.lower)
-+ return NULL;
-+
-+ *iter = lower->list.next;
-+
-+ return lower->private;
-+}
-+EXPORT_SYMBOL(netdev_lower_get_next_private);
-+
-+/**
-+ * netdev_lower_get_next_private_rcu - Get the next ->private from the
-+ * lower neighbour list, RCU
-+ * variant
-+ * @dev: device
-+ * @iter: list_head ** of the current position
-+ *
-+ * Gets the next netdev_adjacent->private from the dev's lower neighbour
-+ * list, starting from iter position. The caller must hold RCU read lock.
-+ */
-+void *netdev_lower_get_next_private_rcu(struct net_device *dev,
-+ struct list_head **iter)
-+{
-+ struct netdev_adjacent *lower;
-+
-+ WARN_ON_ONCE(!rcu_read_lock_held());
-+
-+ lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
-+
-+ if (&lower->list == &dev->adj_list.lower)
-+ return NULL;
-+
-+ *iter = &lower->list;
-+
-+ return lower->private;
-+}
-+EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
-+
-+/**
-+ * netdev_lower_get_next - Get the next device from the lower neighbour
-+ * list
-+ * @dev: device
-+ * @iter: list_head ** of the current position
-+ *
-+ * Gets the next netdev_adjacent from the dev's lower neighbour
-+ * list, starting from iter position. The caller must hold RTNL lock or
-+ * its own locking that guarantees that the neighbour lower
-+ * list will remain unchainged.
-+ */
-+void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
-+{
-+ struct netdev_adjacent *lower;
-+
-+ lower = list_entry((*iter)->next, struct netdev_adjacent, list);
-+
-+ if (&lower->list == &dev->adj_list.lower)
-+ return NULL;
-+
-+ *iter = &lower->list;
-+
-+ return lower->dev;
-+}
-+EXPORT_SYMBOL(netdev_lower_get_next);
-+
-+/**
-+ * netdev_lower_get_first_private_rcu - Get the first ->private from the
-+ * lower neighbour list, RCU
-+ * variant
-+ * @dev: device
-+ *
-+ * Gets the first netdev_adjacent->private from the dev's lower neighbour
-+ * list. The caller must hold RCU read lock.
-+ */
-+void *netdev_lower_get_first_private_rcu(struct net_device *dev)
-+{
-+ struct netdev_adjacent *lower;
-+
-+ lower = list_first_or_null_rcu(&dev->adj_list.lower,
-+ struct netdev_adjacent, list);
-+ if (lower)
-+ return lower->private;
-+ return NULL;
-+}
-+EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
-+
-+/**
-+ * netdev_master_upper_dev_get_rcu - Get master upper device
-+ * @dev: device
-+ *
-+ * Find a master upper device and return pointer to it or NULL in case
-+ * it's not there. The caller must hold the RCU read lock.
-+ */
-+struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
-+{
-+ struct netdev_adjacent *upper;
-+
-+ upper = list_first_or_null_rcu(&dev->adj_list.upper,
-+ struct netdev_adjacent, list);
-+ if (upper && likely(upper->master))
-+ return upper->dev;
-+ return NULL;
-+}
-+EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
-+
-+static int netdev_adjacent_sysfs_add(struct net_device *dev,
-+ struct net_device *adj_dev,
-+ struct list_head *dev_list)
-+{
-+ char linkname[IFNAMSIZ+7];
-+ sprintf(linkname, dev_list == &dev->adj_list.upper ?
-+ "upper_%s" : "lower_%s", adj_dev->name);
-+ return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
-+ linkname);
-+}
-+static void netdev_adjacent_sysfs_del(struct net_device *dev,
-+ char *name,
-+ struct list_head *dev_list)
-+{
-+ char linkname[IFNAMSIZ+7];
-+ sprintf(linkname, dev_list == &dev->adj_list.upper ?
-+ "upper_%s" : "lower_%s", name);
-+ sysfs_remove_link(&(dev->dev.kobj), linkname);
-+}
-+
-+static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
-+ struct net_device *adj_dev,
-+ struct list_head *dev_list)
-+{
-+ return (dev_list == &dev->adj_list.upper ||
-+ dev_list == &dev->adj_list.lower) &&
-+ net_eq(dev_net(dev), dev_net(adj_dev));
-+}
-+
-+static int __netdev_adjacent_dev_insert(struct net_device *dev,
-+ struct net_device *adj_dev,
-+ struct list_head *dev_list,
-+ void *private, bool master)
-+{
-+ struct netdev_adjacent *adj;
-+ int ret;
-+
-+ adj = __netdev_find_adj(dev, adj_dev, dev_list);
-+
-+ if (adj) {
-+ adj->ref_nr++;
-+ return 0;
-+ }
-+
-+ adj = kmalloc(sizeof(*adj), GFP_KERNEL);
-+ if (!adj)
-+ return -ENOMEM;
-+
-+ adj->dev = adj_dev;
-+ adj->master = master;
-+ adj->ref_nr = 1;
-+ adj->private = private;
-+ dev_hold(adj_dev);
-+
-+ pr_debug("dev_hold for %s, because of link added from %s to %s\n",
-+ adj_dev->name, dev->name, adj_dev->name);
-+
-+ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
-+ ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
-+ if (ret)
-+ goto free_adj;
-+ }
-+
-+ /* Ensure that master link is always the first item in list. */
-+ if (master) {
-+ ret = sysfs_create_link(&(dev->dev.kobj),
-+ &(adj_dev->dev.kobj), "master");
-+ if (ret)
-+ goto remove_symlinks;
-+
-+ list_add_rcu(&adj->list, dev_list);
-+ } else {
-+ list_add_tail_rcu(&adj->list, dev_list);
-+ }
-+
-+ return 0;
-+
-+remove_symlinks:
-+ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
-+ netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
-+free_adj:
-+ kfree(adj);
-+ dev_put(adj_dev);
-+
-+ return ret;
-+}
-+
-+static void __netdev_adjacent_dev_remove(struct net_device *dev,
-+ struct net_device *adj_dev,
-+ struct list_head *dev_list)
-+{
-+ struct netdev_adjacent *adj;
-+
-+ adj = __netdev_find_adj(dev, adj_dev, dev_list);
-+
-+ if (!adj) {
-+ pr_err("tried to remove device %s from %s\n",
-+ dev->name, adj_dev->name);
-+ BUG();
-+ }
-+
-+ if (adj->ref_nr > 1) {
-+ pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
-+ adj->ref_nr-1);
-+ adj->ref_nr--;
-+ return;
-+ }
-+
-+ if (adj->master)
-+ sysfs_remove_link(&(dev->dev.kobj), "master");
-+
-+ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
-+ netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
-+
-+ list_del_rcu(&adj->list);
-+ pr_debug("dev_put for %s, because link removed from %s to %s\n",
-+ adj_dev->name, dev->name, adj_dev->name);
-+ dev_put(adj_dev);
-+ kfree_rcu(adj, rcu);
-+}
-+
-+static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
-+ struct net_device *upper_dev,
-+ struct list_head *up_list,
-+ struct list_head *down_list,
-+ void *private, bool master)
-+{
-+ int ret;
-+
-+ ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
-+ master);
-+ if (ret)
-+ return ret;
-+
-+ ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
-+ false);
-+ if (ret) {
-+ __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
-+ return ret;
-+ }
-+
-+ return 0;
-+}
-+
-+static int __netdev_adjacent_dev_link(struct net_device *dev,
-+ struct net_device *upper_dev)
-+{
-+ return __netdev_adjacent_dev_link_lists(dev, upper_dev,
-+ &dev->all_adj_list.upper,
-+ &upper_dev->all_adj_list.lower,
-+ NULL, false);
-+}
-+
-+static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
-+ struct net_device *upper_dev,
-+ struct list_head *up_list,
-+ struct list_head *down_list)
-+{
-+ __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
-+ __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
-+}
-+
-+static void __netdev_adjacent_dev_unlink(struct net_device *dev,
-+ struct net_device *upper_dev)
-+{
-+ __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
-+ &dev->all_adj_list.upper,
-+ &upper_dev->all_adj_list.lower);
-+}
-+
-+static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
-+ struct net_device *upper_dev,
-+ void *private, bool master)
-+{
-+ int ret = __netdev_adjacent_dev_link(dev, upper_dev);
-+
-+ if (ret)
-+ return ret;
-+
-+ ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
-+ &dev->adj_list.upper,
-+ &upper_dev->adj_list.lower,
-+ private, master);
-+ if (ret) {
-+ __netdev_adjacent_dev_unlink(dev, upper_dev);
-+ return ret;
-+ }
-+
-+ return 0;
-+}
-+
-+static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
-+ struct net_device *upper_dev)
-+{
-+ __netdev_adjacent_dev_unlink(dev, upper_dev);
-+ __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
-+ &dev->adj_list.upper,
-+ &upper_dev->adj_list.lower);
-+}
-+
-+static int __netdev_upper_dev_link(struct net_device *dev,
-+ struct net_device *upper_dev, bool master,
-+ void *private)
-+{
-+ struct netdev_adjacent *i, *j, *to_i, *to_j;
-+ int ret = 0;
-+
-+ ASSERT_RTNL();
-+
-+ if (dev == upper_dev)
-+ return -EBUSY;
-+
-+ /* To prevent loops, check if dev is not upper device to upper_dev. */
-+ if (__netdev_find_adj(upper_dev, dev, &upper_dev->all_adj_list.upper))
-+ return -EBUSY;
-+
-+ if (__netdev_find_adj(dev, upper_dev, &dev->adj_list.upper))
-+ return -EEXIST;
-+
-+ if (master && netdev_master_upper_dev_get(dev))
-+ return -EBUSY;
-+
-+ ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private,
-+ master);
-+ if (ret)
-+ return ret;
-+
-+ /* Now that we linked these devs, make all the upper_dev's
-+ * all_adj_list.upper visible to every dev's all_adj_list.lower an
-+ * versa, and don't forget the devices itself. All of these
-+ * links are non-neighbours.
-+ */
-+ list_for_each_entry(i, &dev->all_adj_list.lower, list) {
-+ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
-+ pr_debug("Interlinking %s with %s, non-neighbour\n",
-+ i->dev->name, j->dev->name);
-+ ret = __netdev_adjacent_dev_link(i->dev, j->dev);
-+ if (ret)
-+ goto rollback_mesh;
-+ }
-+ }
-+
-+ /* add dev to every upper_dev's upper device */
-+ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
-+ pr_debug("linking %s's upper device %s with %s\n",
-+ upper_dev->name, i->dev->name, dev->name);
-+ ret = __netdev_adjacent_dev_link(dev, i->dev);
-+ if (ret)
-+ goto rollback_upper_mesh;
-+ }
-+
-+ /* add upper_dev to every dev's lower device */
-+ list_for_each_entry(i, &dev->all_adj_list.lower, list) {
-+ pr_debug("linking %s's lower device %s with %s\n", dev->name,
-+ i->dev->name, upper_dev->name);
-+ ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
-+ if (ret)
-+ goto rollback_lower_mesh;
-+ }
-+
-+ call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
-+ return 0;
-+
-+rollback_lower_mesh:
-+ to_i = i;
-+ list_for_each_entry(i, &dev->all_adj_list.lower, list) {
-+ if (i == to_i)
-+ break;
-+ __netdev_adjacent_dev_unlink(i->dev, upper_dev);
-+ }
-+
-+ i = NULL;
-+
-+rollback_upper_mesh:
-+ to_i = i;
-+ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
-+ if (i == to_i)
-+ break;
-+ __netdev_adjacent_dev_unlink(dev, i->dev);
-+ }
-+
-+ i = j = NULL;
-+
-+rollback_mesh:
-+ to_i = i;
-+ to_j = j;
-+ list_for_each_entry(i, &dev->all_adj_list.lower, list) {
-+ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
-+ if (i == to_i && j == to_j)
-+ break;
-+ __netdev_adjacent_dev_unlink(i->dev, j->dev);
-+ }
-+ if (i == to_i)
-+ break;
-+ }
-+
-+ __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
-+
-+ return ret;
-+}
-+
-+/**
-+ * netdev_upper_dev_link - Add a link to the upper device
-+ * @dev: device
-+ * @upper_dev: new upper device
-+ *
-+ * Adds a link to device which is upper to this one. The caller must hold
-+ * the RTNL lock. On a failure a negative errno code is returned.
-+ * On success the reference counts are adjusted and the function
-+ * returns zero.
-+ */
-+int netdev_upper_dev_link(struct net_device *dev,
-+ struct net_device *upper_dev)
-+{
-+ return __netdev_upper_dev_link(dev, upper_dev, false, NULL);
-+}
-+EXPORT_SYMBOL(netdev_upper_dev_link);
-+
-+/**
-+ * netdev_master_upper_dev_link - Add a master link to the upper device
-+ * @dev: device
-+ * @upper_dev: new upper device
-+ *
-+ * Adds a link to device which is upper to this one. In this case, only
-+ * one master upper device can be linked, although other non-master devices
-+ * might be linked as well. The caller must hold the RTNL lock.
-+ * On a failure a negative errno code is returned. On success the reference
-+ * counts are adjusted and the function returns zero.
-+ */
-+int netdev_master_upper_dev_link(struct net_device *dev,
-+ struct net_device *upper_dev)
-+{
-+ return __netdev_upper_dev_link(dev, upper_dev, true, NULL);
-+}
-+EXPORT_SYMBOL(netdev_master_upper_dev_link);
-+
-+int netdev_master_upper_dev_link_private(struct net_device *dev,
-+ struct net_device *upper_dev,
-+ void *private)
-+{
-+ return __netdev_upper_dev_link(dev, upper_dev, true, private);
-+}
-+EXPORT_SYMBOL(netdev_master_upper_dev_link_private);
-+
-+/**
-+ * netdev_upper_dev_unlink - Removes a link to upper device
-+ * @dev: device
-+ * @upper_dev: new upper device
-+ *
-+ * Removes a link to device which is upper to this one. The caller must hold
-+ * the RTNL lock.
-+ */
-+void netdev_upper_dev_unlink(struct net_device *dev,
-+ struct net_device *upper_dev)
-+{
-+ struct netdev_adjacent *i, *j;
-+ ASSERT_RTNL();
-+
-+ __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
-+
-+ /* Here is the tricky part. We must remove all dev's lower
-+ * devices from all upper_dev's upper devices and vice
-+ * versa, to maintain the graph relationship.
-+ */
-+ list_for_each_entry(i, &dev->all_adj_list.lower, list)
-+ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
-+ __netdev_adjacent_dev_unlink(i->dev, j->dev);
-+
-+ /* remove also the devices itself from lower/upper device
-+ * list
-+ */
-+ list_for_each_entry(i, &dev->all_adj_list.lower, list)
-+ __netdev_adjacent_dev_unlink(i->dev, upper_dev);
-+
-+ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
-+ __netdev_adjacent_dev_unlink(dev, i->dev);
-+
-+ call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
-+}
-+EXPORT_SYMBOL(netdev_upper_dev_unlink);
-+
-+/**
-+ * netdev_bonding_info_change - Dispatch event about slave change
-+ * @dev: device
-+ * @bonding_info: info to dispatch
-+ *
-+ * Send NETDEV_BONDING_INFO to netdev notifiers with info.
-+ * The caller must hold the RTNL lock.
-+ */
-+void netdev_bonding_info_change(struct net_device *dev,
-+ struct netdev_bonding_info *bonding_info)
-+{
-+ struct netdev_notifier_bonding_info info;
-+
-+ memcpy(&info.bonding_info, bonding_info,
-+ sizeof(struct netdev_bonding_info));
-+ call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
-+ &info.info);
-+}
-+EXPORT_SYMBOL(netdev_bonding_info_change);
-+
-+static void netdev_adjacent_add_links(struct net_device *dev)
-+{
-+ struct netdev_adjacent *iter;
-+
-+ struct net *net = dev_net(dev);
-+
-+ list_for_each_entry(iter, &dev->adj_list.upper, list) {
-+ if (!net_eq(net,dev_net(iter->dev)))
-+ continue;
-+ netdev_adjacent_sysfs_add(iter->dev, dev,
-+ &iter->dev->adj_list.lower);
-+ netdev_adjacent_sysfs_add(dev, iter->dev,
-+ &dev->adj_list.upper);
-+ }
-+
-+ list_for_each_entry(iter, &dev->adj_list.lower, list) {
-+ if (!net_eq(net,dev_net(iter->dev)))
-+ continue;
-+ netdev_adjacent_sysfs_add(iter->dev, dev,
-+ &iter->dev->adj_list.upper);
-+ netdev_adjacent_sysfs_add(dev, iter->dev,
-+ &dev->adj_list.lower);
-+ }
-+}
-+
-+static void netdev_adjacent_del_links(struct net_device *dev)
-+{
-+ struct netdev_adjacent *iter;
-+
-+ struct net *net = dev_net(dev);
-+
-+ list_for_each_entry(iter, &dev->adj_list.upper, list) {
-+ if (!net_eq(net,dev_net(iter->dev)))
-+ continue;
-+ netdev_adjacent_sysfs_del(iter->dev, dev->name,
-+ &iter->dev->adj_list.lower);
-+ netdev_adjacent_sysfs_del(dev, iter->dev->name,
-+ &dev->adj_list.upper);
-+ }
-+
-+ list_for_each_entry(iter, &dev->adj_list.lower, list) {
-+ if (!net_eq(net,dev_net(iter->dev)))
-+ continue;
-+ netdev_adjacent_sysfs_del(iter->dev, dev->name,
-+ &iter->dev->adj_list.upper);
-+ netdev_adjacent_sysfs_del(dev, iter->dev->name,
-+ &dev->adj_list.lower);
-+ }
-+}
-+
-+void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
-+{
-+ struct netdev_adjacent *iter;
-+
-+ struct net *net = dev_net(dev);
-+
-+ list_for_each_entry(iter, &dev->adj_list.upper, list) {
-+ if (!net_eq(net,dev_net(iter->dev)))
-+ continue;
-+ netdev_adjacent_sysfs_del(iter->dev, oldname,
-+ &iter->dev->adj_list.lower);
-+ netdev_adjacent_sysfs_add(iter->dev, dev,
-+ &iter->dev->adj_list.lower);
-+ }
-+
-+ list_for_each_entry(iter, &dev->adj_list.lower, list) {
-+ if (!net_eq(net,dev_net(iter->dev)))
-+ continue;
-+ netdev_adjacent_sysfs_del(iter->dev, oldname,
-+ &iter->dev->adj_list.upper);
-+ netdev_adjacent_sysfs_add(iter->dev, dev,
-+ &iter->dev->adj_list.upper);
-+ }
-+}
-+
-+void *netdev_lower_dev_get_private(struct net_device *dev,
-+ struct net_device *lower_dev)
-+{
-+ struct netdev_adjacent *lower;
-+
-+ if (!lower_dev)
-+ return NULL;
-+ lower = __netdev_find_adj(dev, lower_dev, &dev->adj_list.lower);
-+ if (!lower)
-+ return NULL;
-+
-+ return lower->private;
-+}
-+EXPORT_SYMBOL(netdev_lower_dev_get_private);
-+
-+
-+int dev_get_nest_level(struct net_device *dev,
-+ bool (*type_check)(struct net_device *dev))
-+{
-+ struct net_device *lower = NULL;
-+ struct list_head *iter;
-+ int max_nest = -1;
-+ int nest;
-+
-+ ASSERT_RTNL();
-+
-+ netdev_for_each_lower_dev(dev, lower, iter) {
-+ nest = dev_get_nest_level(lower, type_check);
-+ if (max_nest < nest)
-+ max_nest = nest;
-+ }
-+
-+ if (type_check(dev))
-+ max_nest++;
-+
-+ return max_nest;
-+}
-+EXPORT_SYMBOL(dev_get_nest_level);
-+
-+static void dev_change_rx_flags(struct net_device *dev, int flags)
-+{
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+
-+ if (ops->ndo_change_rx_flags)
-+ ops->ndo_change_rx_flags(dev, flags);
-+}
-+
-+static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
-+{
-+ unsigned int old_flags = dev->flags;
-+ kuid_t uid;
-+ kgid_t gid;
-+
-+ ASSERT_RTNL();
-+
-+ dev->flags |= IFF_PROMISC;
-+ dev->promiscuity += inc;
-+ if (dev->promiscuity == 0) {
-+ /*
-+ * Avoid overflow.
-+ * If inc causes overflow, untouch promisc and return error.
-+ */
-+ if (inc < 0)
-+ dev->flags &= ~IFF_PROMISC;
-+ else {
-+ dev->promiscuity -= inc;
-+ pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
-+ dev->name);
-+ return -EOVERFLOW;
-+ }
-+ }
-+ if (dev->flags != old_flags) {
-+ pr_info("device %s %s promiscuous mode\n",
-+ dev->name,
-+ dev->flags & IFF_PROMISC ? "entered" : "left");
-+ if (audit_enabled) {
-+ current_uid_gid(&uid, &gid);
-+ audit_log(current->audit_context, GFP_ATOMIC,
-+ AUDIT_ANOM_PROMISCUOUS,
-+ "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
-+ dev->name, (dev->flags & IFF_PROMISC),
-+ (old_flags & IFF_PROMISC),
-+ from_kuid(&init_user_ns, audit_get_loginuid(current)),
-+ from_kuid(&init_user_ns, uid),
-+ from_kgid(&init_user_ns, gid),
-+ audit_get_sessionid(current));
-+ }
-+
-+ dev_change_rx_flags(dev, IFF_PROMISC);
-+ }
-+ if (notify)
-+ __dev_notify_flags(dev, old_flags, IFF_PROMISC);
-+ return 0;
-+}
-+
-+/**
-+ * dev_set_promiscuity - update promiscuity count on a device
-+ * @dev: device
-+ * @inc: modifier
-+ *
-+ * Add or remove promiscuity from a device. While the count in the device
-+ * remains above zero the interface remains promiscuous. Once it hits zero
-+ * the device reverts back to normal filtering operation. A negative inc
-+ * value is used to drop promiscuity on the device.
-+ * Return 0 if successful or a negative errno code on error.
-+ */
-+int dev_set_promiscuity(struct net_device *dev, int inc)
-+{
-+ unsigned int old_flags = dev->flags;
-+ int err;
-+
-+ err = __dev_set_promiscuity(dev, inc, true);
-+ if (err < 0)
-+ return err;
-+ if (dev->flags != old_flags)
-+ dev_set_rx_mode(dev);
-+ return err;
-+}
-+EXPORT_SYMBOL(dev_set_promiscuity);
-+
-+static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
-+{
-+ unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
-+
-+ ASSERT_RTNL();
-+
-+ dev->flags |= IFF_ALLMULTI;
-+ dev->allmulti += inc;
-+ if (dev->allmulti == 0) {
-+ /*
-+ * Avoid overflow.
-+ * If inc causes overflow, untouch allmulti and return error.
-+ */
-+ if (inc < 0)
-+ dev->flags &= ~IFF_ALLMULTI;
-+ else {
-+ dev->allmulti -= inc;
-+ pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
-+ dev->name);
-+ return -EOVERFLOW;
-+ }
-+ }
-+ if (dev->flags ^ old_flags) {
-+ dev_change_rx_flags(dev, IFF_ALLMULTI);
-+ dev_set_rx_mode(dev);
-+ if (notify)
-+ __dev_notify_flags(dev, old_flags,
-+ dev->gflags ^ old_gflags);
-+ }
-+ return 0;
-+}
-+
-+/**
-+ * dev_set_allmulti - update allmulti count on a device
-+ * @dev: device
-+ * @inc: modifier
-+ *
-+ * Add or remove reception of all multicast frames to a device. While the
-+ * count in the device remains above zero the interface remains listening
-+ * to all interfaces. Once it hits zero the device reverts back to normal
-+ * filtering operation. A negative @inc value is used to drop the counter
-+ * when releasing a resource needing all multicasts.
-+ * Return 0 if successful or a negative errno code on error.
-+ */
-+
-+int dev_set_allmulti(struct net_device *dev, int inc)
-+{
-+ return __dev_set_allmulti(dev, inc, true);
-+}
-+EXPORT_SYMBOL(dev_set_allmulti);
-+
-+/*
-+ * Upload unicast and multicast address lists to device and
-+ * configure RX filtering. When the device doesn't support unicast
-+ * filtering it is put in promiscuous mode while unicast addresses
-+ * are present.
-+ */
-+void __dev_set_rx_mode(struct net_device *dev)
-+{
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+
-+ /* dev_open will call this function so the list will stay sane. */
-+ if (!(dev->flags&IFF_UP))
-+ return;
-+
-+ if (!netif_device_present(dev))
-+ return;
-+
-+ if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
-+ /* Unicast addresses changes may only happen under the rtnl,
-+ * therefore calling __dev_set_promiscuity here is safe.
-+ */
-+ if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
-+ __dev_set_promiscuity(dev, 1, false);
-+ dev->uc_promisc = true;
-+ } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
-+ __dev_set_promiscuity(dev, -1, false);
-+ dev->uc_promisc = false;
-+ }
-+ }
-+
-+ if (ops->ndo_set_rx_mode)
-+ ops->ndo_set_rx_mode(dev);
-+}
-+
-+void dev_set_rx_mode(struct net_device *dev)
-+{
-+ netif_addr_lock_bh(dev);
-+ __dev_set_rx_mode(dev);
-+ netif_addr_unlock_bh(dev);
-+}
-+
-+/**
-+ * dev_get_flags - get flags reported to userspace
-+ * @dev: device
-+ *
-+ * Get the combination of flag bits exported through APIs to userspace.
-+ */
-+unsigned int dev_get_flags(const struct net_device *dev)
-+{
-+ unsigned int flags;
-+
-+ flags = (dev->flags & ~(IFF_PROMISC |
-+ IFF_ALLMULTI |
-+ IFF_RUNNING |
-+ IFF_LOWER_UP |
-+ IFF_DORMANT)) |
-+ (dev->gflags & (IFF_PROMISC |
-+ IFF_ALLMULTI));
-+
-+ if (netif_running(dev)) {
-+ if (netif_oper_up(dev))
-+ flags |= IFF_RUNNING;
-+ if (netif_carrier_ok(dev))
-+ flags |= IFF_LOWER_UP;
-+ if (netif_dormant(dev))
-+ flags |= IFF_DORMANT;
-+ }
-+
-+ return flags;
-+}
-+EXPORT_SYMBOL(dev_get_flags);
-+
-+int __dev_change_flags(struct net_device *dev, unsigned int flags)
-+{
-+ unsigned int old_flags = dev->flags;
-+ int ret;
-+
-+ ASSERT_RTNL();
-+
-+ /*
-+ * Set the flags on our device.
-+ */
-+
-+ dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
-+ IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
-+ IFF_AUTOMEDIA)) |
-+ (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
-+ IFF_ALLMULTI));
-+
-+ /*
-+ * Load in the correct multicast list now the flags have changed.
-+ */
-+
-+ if ((old_flags ^ flags) & IFF_MULTICAST)
-+ dev_change_rx_flags(dev, IFF_MULTICAST);
-+
-+ dev_set_rx_mode(dev);
-+
-+ /*
-+ * Have we downed the interface. We handle IFF_UP ourselves
-+ * according to user attempts to set it, rather than blindly
-+ * setting it.
-+ */
-+
-+ ret = 0;
-+ if ((old_flags ^ flags) & IFF_UP)
-+ ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
-+
-+ if ((flags ^ dev->gflags) & IFF_PROMISC) {
-+ int inc = (flags & IFF_PROMISC) ? 1 : -1;
-+ unsigned int old_flags = dev->flags;
-+
-+ dev->gflags ^= IFF_PROMISC;
-+
-+ if (__dev_set_promiscuity(dev, inc, false) >= 0)
-+ if (dev->flags != old_flags)
-+ dev_set_rx_mode(dev);
-+ }
-+
-+ /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
-+ is important. Some (broken) drivers set IFF_PROMISC, when
-+ IFF_ALLMULTI is requested not asking us and not reporting.
-+ */
-+ if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
-+ int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
-+
-+ dev->gflags ^= IFF_ALLMULTI;
-+ __dev_set_allmulti(dev, inc, false);
-+ }
-+
-+ return ret;
-+}
-+
-+void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
-+ unsigned int gchanges)
-+{
-+ unsigned int changes = dev->flags ^ old_flags;
-+
-+ if (gchanges)
-+ rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
-+
-+ if (changes & IFF_UP) {
-+ if (dev->flags & IFF_UP)
-+ call_netdevice_notifiers(NETDEV_UP, dev);
-+ else
-+ call_netdevice_notifiers(NETDEV_DOWN, dev);
-+ }
-+
-+ if (dev->flags & IFF_UP &&
-+ (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
-+ struct netdev_notifier_change_info change_info;
-+
-+ change_info.flags_changed = changes;
-+ call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
-+ &change_info.info);
-+ }
-+}
-+
-+/**
-+ * dev_change_flags - change device settings
-+ * @dev: device
-+ * @flags: device state flags
-+ *
-+ * Change settings on device based state flags. The flags are
-+ * in the userspace exported format.
-+ */
-+int dev_change_flags(struct net_device *dev, unsigned int flags)
-+{
-+ int ret;
-+ unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
-+
-+ ret = __dev_change_flags(dev, flags);
-+ if (ret < 0)
-+ return ret;
-+
-+ changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
-+ __dev_notify_flags(dev, old_flags, changes);
-+ return ret;
-+}
-+EXPORT_SYMBOL(dev_change_flags);
-+
-+static int __dev_set_mtu(struct net_device *dev, int new_mtu)
-+{
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+
-+ if (ops->ndo_change_mtu)
-+ return ops->ndo_change_mtu(dev, new_mtu);
-+
-+ dev->mtu = new_mtu;
-+ return 0;
-+}
-+
-+/**
-+ * dev_set_mtu - Change maximum transfer unit
-+ * @dev: device
-+ * @new_mtu: new transfer unit
-+ *
-+ * Change the maximum transfer size of the network device.
-+ */
-+int dev_set_mtu(struct net_device *dev, int new_mtu)
-+{
-+ int err, orig_mtu;
-+
-+ if (new_mtu == dev->mtu)
-+ return 0;
-+
-+ /* MTU must be positive. */
-+ if (new_mtu < 0)
-+ return -EINVAL;
-+
-+ if (!netif_device_present(dev))
-+ return -ENODEV;
-+
-+ err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
-+ err = notifier_to_errno(err);
-+ if (err)
-+ return err;
-+
-+ orig_mtu = dev->mtu;
-+ err = __dev_set_mtu(dev, new_mtu);
-+
-+ if (!err) {
-+ err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
-+ err = notifier_to_errno(err);
-+ if (err) {
-+ /* setting mtu back and notifying everyone again,
-+ * so that they have a chance to revert changes.
-+ */
-+ __dev_set_mtu(dev, orig_mtu);
-+ call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
-+ }
-+ }
-+ return err;
-+}
-+EXPORT_SYMBOL(dev_set_mtu);
-+
-+/**
-+ * dev_set_group - Change group this device belongs to
-+ * @dev: device
-+ * @new_group: group this device should belong to
-+ */
-+void dev_set_group(struct net_device *dev, int new_group)
-+{
-+ dev->group = new_group;
-+}
-+EXPORT_SYMBOL(dev_set_group);
-+
-+/**
-+ * dev_set_mac_address - Change Media Access Control Address
-+ * @dev: device
-+ * @sa: new address
-+ *
-+ * Change the hardware (MAC) address of the device
-+ */
-+int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
-+{
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+ int err;
-+
-+ if (!ops->ndo_set_mac_address)
-+ return -EOPNOTSUPP;
-+ if (sa->sa_family != dev->type)
-+ return -EINVAL;
-+ if (!netif_device_present(dev))
-+ return -ENODEV;
-+ err = ops->ndo_set_mac_address(dev, sa);
-+ if (err)
-+ return err;
-+ dev->addr_assign_type = NET_ADDR_SET;
-+ call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
-+ add_device_randomness(dev->dev_addr, dev->addr_len);
-+ return 0;
-+}
-+EXPORT_SYMBOL(dev_set_mac_address);
-+
-+/**
-+ * dev_change_carrier - Change device carrier
-+ * @dev: device
-+ * @new_carrier: new value
-+ *
-+ * Change device carrier
-+ */
-+int dev_change_carrier(struct net_device *dev, bool new_carrier)
-+{
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+
-+ if (!ops->ndo_change_carrier)
-+ return -EOPNOTSUPP;
-+ if (!netif_device_present(dev))
-+ return -ENODEV;
-+ return ops->ndo_change_carrier(dev, new_carrier);
-+}
-+EXPORT_SYMBOL(dev_change_carrier);
-+
-+/**
-+ * dev_get_phys_port_id - Get device physical port ID
-+ * @dev: device
-+ * @ppid: port ID
-+ *
-+ * Get device physical port ID
-+ */
-+int dev_get_phys_port_id(struct net_device *dev,
-+ struct netdev_phys_item_id *ppid)
-+{
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+
-+ if (!ops->ndo_get_phys_port_id)
-+ return -EOPNOTSUPP;
-+ return ops->ndo_get_phys_port_id(dev, ppid);
-+}
-+EXPORT_SYMBOL(dev_get_phys_port_id);
-+
-+/**
-+ * dev_get_phys_port_name - Get device physical port name
-+ * @dev: device
-+ * @name: port name
-+ *
-+ * Get device physical port name
-+ */
-+int dev_get_phys_port_name(struct net_device *dev,
-+ char *name, size_t len)
-+{
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+
-+ if (!ops->ndo_get_phys_port_name)
-+ return -EOPNOTSUPP;
-+ return ops->ndo_get_phys_port_name(dev, name, len);
-+}
-+EXPORT_SYMBOL(dev_get_phys_port_name);
-+
-+/**
-+ * dev_new_index - allocate an ifindex
-+ * @net: the applicable net namespace
-+ *
-+ * Returns a suitable unique value for a new device interface
-+ * number. The caller must hold the rtnl semaphore or the
-+ * dev_base_lock to be sure it remains unique.
-+ */
-+static int dev_new_index(struct net *net)
-+{
-+ int ifindex = net->ifindex;
-+ for (;;) {
-+ if (++ifindex <= 0)
-+ ifindex = 1;
-+ if (!__dev_get_by_index(net, ifindex))
-+ return net->ifindex = ifindex;
-+ }
-+}
-+
-+/* Delayed registration/unregisteration */
-+static LIST_HEAD(net_todo_list);
-+DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
-+
-+static void net_set_todo(struct net_device *dev)
-+{
-+ list_add_tail(&dev->todo_list, &net_todo_list);
-+ dev_net(dev)->dev_unreg_count++;
-+}
-+
-+static void rollback_registered_many(struct list_head *head)
-+{
-+ struct net_device *dev, *tmp;
-+ LIST_HEAD(close_head);
-+
-+ BUG_ON(dev_boot_phase);
-+ ASSERT_RTNL();
-+
-+ list_for_each_entry_safe(dev, tmp, head, unreg_list) {
-+ /* Some devices call without registering
-+ * for initialization unwind. Remove those
-+ * devices and proceed with the remaining.
-+ */
-+ if (dev->reg_state == NETREG_UNINITIALIZED) {
-+ pr_debug("unregister_netdevice: device %s/%p never was registered\n",
-+ dev->name, dev);
-+
-+ WARN_ON(1);
-+ list_del(&dev->unreg_list);
-+ continue;
-+ }
-+ dev->dismantle = true;
-+ BUG_ON(dev->reg_state != NETREG_REGISTERED);
-+ }
-+
-+ /* If device is running, close it first. */
-+ list_for_each_entry(dev, head, unreg_list)
-+ list_add_tail(&dev->close_list, &close_head);
-+ dev_close_many(&close_head, true);
-+
-+ list_for_each_entry(dev, head, unreg_list) {
-+ /* And unlink it from device chain. */
-+ unlist_netdevice(dev);
-+
-+ dev->reg_state = NETREG_UNREGISTERING;
-+ on_each_cpu(flush_backlog, dev, 1);
-+ }
-+
-+ synchronize_net();
-+
-+ list_for_each_entry(dev, head, unreg_list) {
-+ struct sk_buff *skb = NULL;
-+
-+ /* Shutdown queueing discipline. */
-+ dev_shutdown(dev);
-+
-+
-+ /* Notify protocols, that we are about to destroy
-+ this device. They should clean all the things.
-+ */
-+ call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
-+
-+ if (!dev->rtnl_link_ops ||
-+ dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
-+ skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
-+ GFP_KERNEL);
-+
-+ /*
-+ * Flush the unicast and multicast chains
-+ */
-+ dev_uc_flush(dev);
-+ dev_mc_flush(dev);
-+
-+ if (dev->netdev_ops->ndo_uninit)
-+ dev->netdev_ops->ndo_uninit(dev);
-+
-+ if (skb)
-+ rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
-+
-+ /* Notifier chain MUST detach us all upper devices. */
-+ WARN_ON(netdev_has_any_upper_dev(dev));
-+
-+ /* Remove entries from kobject tree */
-+ netdev_unregister_kobject(dev);
-+#ifdef CONFIG_XPS
-+ /* Remove XPS queueing entries */
-+ netif_reset_xps_queues_gt(dev, 0);
-+#endif
-+ }
-+
-+ synchronize_net();
-+
-+ list_for_each_entry(dev, head, unreg_list)
-+ dev_put(dev);
-+}
-+
-+static void rollback_registered(struct net_device *dev)
-+{
-+ LIST_HEAD(single);
-+
-+ list_add(&dev->unreg_list, &single);
-+ rollback_registered_many(&single);
-+ list_del(&single);
-+}
-+
-+static netdev_features_t netdev_fix_features(struct net_device *dev,
-+ netdev_features_t features)
-+{
-+ /* Fix illegal checksum combinations */
-+ if ((features & NETIF_F_HW_CSUM) &&
-+ (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
-+ netdev_warn(dev, "mixed HW and IP checksum settings.\n");
-+ features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
-+ }
-+
-+ /* TSO requires that SG is present as well. */
-+ if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
-+ netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
-+ features &= ~NETIF_F_ALL_TSO;
-+ }
-+
-+ if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
-+ !(features & NETIF_F_IP_CSUM)) {
-+ netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
-+ features &= ~NETIF_F_TSO;
-+ features &= ~NETIF_F_TSO_ECN;
-+ }
-+
-+ if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
-+ !(features & NETIF_F_IPV6_CSUM)) {
-+ netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
-+ features &= ~NETIF_F_TSO6;
-+ }
-+
-+ /* TSO ECN requires that TSO is present as well. */
-+ if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
-+ features &= ~NETIF_F_TSO_ECN;
-+
-+ /* Software GSO depends on SG. */
-+ if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
-+ netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
-+ features &= ~NETIF_F_GSO;
-+ }
-+
-+ /* UFO needs SG and checksumming */
-+ if (features & NETIF_F_UFO) {
-+ /* maybe split UFO into V4 and V6? */
-+ if (!((features & NETIF_F_GEN_CSUM) ||
-+ (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
-+ == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
-+ netdev_dbg(dev,
-+ "Dropping NETIF_F_UFO since no checksum offload features.\n");
-+ features &= ~NETIF_F_UFO;
-+ }
-+
-+ if (!(features & NETIF_F_SG)) {
-+ netdev_dbg(dev,
-+ "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
-+ features &= ~NETIF_F_UFO;
-+ }
-+ }
-+
-+#ifdef CONFIG_NET_RX_BUSY_POLL
-+ if (dev->netdev_ops->ndo_busy_poll)
-+ features |= NETIF_F_BUSY_POLL;
-+ else
-+#endif
-+ features &= ~NETIF_F_BUSY_POLL;
-+
-+ return features;
-+}
-+
-+int __netdev_update_features(struct net_device *dev)
-+{
-+ netdev_features_t features;
-+ int err = 0;
-+
-+ ASSERT_RTNL();
-+
-+ features = netdev_get_wanted_features(dev);
-+
-+ if (dev->netdev_ops->ndo_fix_features)
-+ features = dev->netdev_ops->ndo_fix_features(dev, features);
-+
-+ /* driver might be less strict about feature dependencies */
-+ features = netdev_fix_features(dev, features);
-+
-+ if (dev->features == features)
-+ return 0;
-+
-+ netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
-+ &dev->features, &features);
-+
-+ if (dev->netdev_ops->ndo_set_features)
-+ err = dev->netdev_ops->ndo_set_features(dev, features);
-+
-+ if (unlikely(err < 0)) {
-+ netdev_err(dev,
-+ "set_features() failed (%d); wanted %pNF, left %pNF\n",
-+ err, &features, &dev->features);
-+ return -1;
-+ }
-+
-+ if (!err)
-+ dev->features = features;
-+
-+ return 1;
-+}
-+
-+/**
-+ * netdev_update_features - recalculate device features
-+ * @dev: the device to check
-+ *
-+ * Recalculate dev->features set and send notifications if it
-+ * has changed. Should be called after driver or hardware dependent
-+ * conditions might have changed that influence the features.
-+ */
-+void netdev_update_features(struct net_device *dev)
-+{
-+ if (__netdev_update_features(dev))
-+ netdev_features_change(dev);
-+}
-+EXPORT_SYMBOL(netdev_update_features);
-+
-+/**
-+ * netdev_change_features - recalculate device features
-+ * @dev: the device to check
-+ *
-+ * Recalculate dev->features set and send notifications even
-+ * if they have not changed. Should be called instead of
-+ * netdev_update_features() if also dev->vlan_features might
-+ * have changed to allow the changes to be propagated to stacked
-+ * VLAN devices.
-+ */
-+void netdev_change_features(struct net_device *dev)
-+{
-+ __netdev_update_features(dev);
-+ netdev_features_change(dev);
-+}
-+EXPORT_SYMBOL(netdev_change_features);
-+
-+/**
-+ * netif_stacked_transfer_operstate - transfer operstate
-+ * @rootdev: the root or lower level device to transfer state from
-+ * @dev: the device to transfer operstate to
-+ *
-+ * Transfer operational state from root to device. This is normally
-+ * called when a stacking relationship exists between the root
-+ * device and the device(a leaf device).
-+ */
-+void netif_stacked_transfer_operstate(const struct net_device *rootdev,
-+ struct net_device *dev)
-+{
-+ if (rootdev->operstate == IF_OPER_DORMANT)
-+ netif_dormant_on(dev);
-+ else
-+ netif_dormant_off(dev);
-+
-+ if (netif_carrier_ok(rootdev)) {
-+ if (!netif_carrier_ok(dev))
-+ netif_carrier_on(dev);
-+ } else {
-+ if (netif_carrier_ok(dev))
-+ netif_carrier_off(dev);
-+ }
-+}
-+EXPORT_SYMBOL(netif_stacked_transfer_operstate);
-+
-+#ifdef CONFIG_SYSFS
-+static int netif_alloc_rx_queues(struct net_device *dev)
-+{
-+ unsigned int i, count = dev->num_rx_queues;
-+ struct netdev_rx_queue *rx;
-+ size_t sz = count * sizeof(*rx);
-+
-+ BUG_ON(count < 1);
-+
-+ rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
-+ if (!rx) {
-+ rx = vzalloc(sz);
-+ if (!rx)
-+ return -ENOMEM;
-+ }
-+ dev->_rx = rx;
-+
-+ for (i = 0; i < count; i++)
-+ rx[i].dev = dev;
-+ return 0;
-+}
-+#endif
-+
-+static void netdev_init_one_queue(struct net_device *dev,
-+ struct netdev_queue *queue, void *_unused)
-+{
-+ /* Initialize queue lock */
-+ spin_lock_init(&queue->_xmit_lock);
-+ netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
-+ queue->xmit_lock_owner = -1;
-+ netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
-+ queue->dev = dev;
-+#ifdef CONFIG_BQL
-+ dql_init(&queue->dql, HZ);
-+#endif
-+}
-+
-+static void netif_free_tx_queues(struct net_device *dev)
-+{
-+ kvfree(dev->_tx);
-+}
-+
-+static int netif_alloc_netdev_queues(struct net_device *dev)
-+{
-+ unsigned int count = dev->num_tx_queues;
-+ struct netdev_queue *tx;
-+ size_t sz = count * sizeof(*tx);
-+
-+ if (count < 1 || count > 0xffff)
-+ return -EINVAL;
-+
-+ tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
-+ if (!tx) {
-+ tx = vzalloc(sz);
-+ if (!tx)
-+ return -ENOMEM;
-+ }
-+ dev->_tx = tx;
-+
-+ netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
-+ spin_lock_init(&dev->tx_global_lock);
-+
-+ return 0;
-+}
-+
-+/**
-+ * register_netdevice - register a network device
-+ * @dev: device to register
-+ *
-+ * Take a completed network device structure and add it to the kernel
-+ * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
-+ * chain. 0 is returned on success. A negative errno code is returned
-+ * on a failure to set up the device, or if the name is a duplicate.
-+ *
-+ * Callers must hold the rtnl semaphore. You may want
-+ * register_netdev() instead of this.
-+ *
-+ * BUGS:
-+ * The locking appears insufficient to guarantee two parallel registers
-+ * will not get the same name.
-+ */
-+
-+int register_netdevice(struct net_device *dev)
-+{
-+ int ret;
-+ struct net *net = dev_net(dev);
-+
-+ BUG_ON(dev_boot_phase);
-+ ASSERT_RTNL();
-+
-+ might_sleep();
-+
-+ /* When net_device's are persistent, this will be fatal. */
-+ BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
-+ BUG_ON(!net);
-+
-+ spin_lock_init(&dev->addr_list_lock);
-+ netdev_set_addr_lockdep_class(dev);
-+
-+ ret = dev_get_valid_name(net, dev, dev->name);
-+ if (ret < 0)
-+ goto out;
-+
-+ /* Init, if this function is available */
-+ if (dev->netdev_ops->ndo_init) {
-+ ret = dev->netdev_ops->ndo_init(dev);
-+ if (ret) {
-+ if (ret > 0)
-+ ret = -EIO;
-+ goto out;
-+ }
-+ }
-+
-+ if (((dev->hw_features | dev->features) &
-+ NETIF_F_HW_VLAN_CTAG_FILTER) &&
-+ (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
-+ !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
-+ netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
-+ ret = -EINVAL;
-+ goto err_uninit;
-+ }
-+
-+ ret = -EBUSY;
-+ if (!dev->ifindex)
-+ dev->ifindex = dev_new_index(net);
-+ else if (__dev_get_by_index(net, dev->ifindex))
-+ goto err_uninit;
-+
-+ /* Transfer changeable features to wanted_features and enable
-+ * software offloads (GSO and GRO).
-+ */
-+ dev->hw_features |= NETIF_F_SOFT_FEATURES;
-+ dev->features |= NETIF_F_SOFT_FEATURES;
-+ dev->wanted_features = dev->features & dev->hw_features;
-+
-+ if (!(dev->flags & IFF_LOOPBACK)) {
-+ dev->hw_features |= NETIF_F_NOCACHE_COPY;
-+ }
-+
-+ /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
-+ */
-+ dev->vlan_features |= NETIF_F_HIGHDMA;
-+
-+ /* Make NETIF_F_SG inheritable to tunnel devices.
-+ */
-+ dev->hw_enc_features |= NETIF_F_SG;
-+
-+ /* Make NETIF_F_SG inheritable to MPLS.
-+ */
-+ dev->mpls_features |= NETIF_F_SG;
-+
-+ ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
-+ ret = notifier_to_errno(ret);
-+ if (ret)
-+ goto err_uninit;
-+
-+ ret = netdev_register_kobject(dev);
-+ if (ret)
-+ goto err_uninit;
-+ dev->reg_state = NETREG_REGISTERED;
-+
-+ __netdev_update_features(dev);
-+
-+ /*
-+ * Default initial state at registry is that the
-+ * device is present.
-+ */
-+
-+ set_bit(__LINK_STATE_PRESENT, &dev->state);
-+
-+ linkwatch_init_dev(dev);
-+
-+ dev_init_scheduler(dev);
-+ dev_hold(dev);
-+ list_netdevice(dev);
-+ add_device_randomness(dev->dev_addr, dev->addr_len);
-+
-+ /* If the device has permanent device address, driver should
-+ * set dev_addr and also addr_assign_type should be set to
-+ * NET_ADDR_PERM (default value).
-+ */
-+ if (dev->addr_assign_type == NET_ADDR_PERM)
-+ memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
-+
-+ /* Notify protocols, that a new device appeared. */
-+ ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
-+ ret = notifier_to_errno(ret);
-+ if (ret) {
-+ rollback_registered(dev);
-+ dev->reg_state = NETREG_UNREGISTERED;
-+ }
-+ /*
-+ * Prevent userspace races by waiting until the network
-+ * device is fully setup before sending notifications.
-+ */
-+ if (!dev->rtnl_link_ops ||
-+ dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
-+ rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
-+
-+out:
-+ return ret;
-+
-+err_uninit:
-+ if (dev->netdev_ops->ndo_uninit)
-+ dev->netdev_ops->ndo_uninit(dev);
-+ goto out;
-+}
-+EXPORT_SYMBOL(register_netdevice);
-+
-+/**
-+ * init_dummy_netdev - init a dummy network device for NAPI
-+ * @dev: device to init
-+ *
-+ * This takes a network device structure and initialize the minimum
-+ * amount of fields so it can be used to schedule NAPI polls without
-+ * registering a full blown interface. This is to be used by drivers
-+ * that need to tie several hardware interfaces to a single NAPI
-+ * poll scheduler due to HW limitations.
-+ */
-+int init_dummy_netdev(struct net_device *dev)
-+{
-+ /* Clear everything. Note we don't initialize spinlocks
-+ * are they aren't supposed to be taken by any of the
-+ * NAPI code and this dummy netdev is supposed to be
-+ * only ever used for NAPI polls
-+ */
-+ memset(dev, 0, sizeof(struct net_device));
-+
-+ /* make sure we BUG if trying to hit standard
-+ * register/unregister code path
-+ */
-+ dev->reg_state = NETREG_DUMMY;
-+
-+ /* NAPI wants this */
-+ INIT_LIST_HEAD(&dev->napi_list);
-+
-+ /* a dummy interface is started by default */
-+ set_bit(__LINK_STATE_PRESENT, &dev->state);
-+ set_bit(__LINK_STATE_START, &dev->state);
-+
-+ /* Note : We dont allocate pcpu_refcnt for dummy devices,
-+ * because users of this 'device' dont need to change
-+ * its refcount.
-+ */
-+
-+ return 0;
-+}
-+EXPORT_SYMBOL_GPL(init_dummy_netdev);
-+
-+
-+/**
-+ * register_netdev - register a network device
-+ * @dev: device to register
-+ *
-+ * Take a completed network device structure and add it to the kernel
-+ * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
-+ * chain. 0 is returned on success. A negative errno code is returned
-+ * on a failure to set up the device, or if the name is a duplicate.
-+ *
-+ * This is a wrapper around register_netdevice that takes the rtnl semaphore
-+ * and expands the device name if you passed a format string to
-+ * alloc_netdev.
-+ */
-+int register_netdev(struct net_device *dev)
-+{
-+ int err;
-+
-+ rtnl_lock();
-+ err = register_netdevice(dev);
-+ rtnl_unlock();
-+ return err;
-+}
-+EXPORT_SYMBOL(register_netdev);
-+
-+int netdev_refcnt_read(const struct net_device *dev)
-+{
-+ int i, refcnt = 0;
-+
-+ for_each_possible_cpu(i)
-+ refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
-+ return refcnt;
-+}
-+EXPORT_SYMBOL(netdev_refcnt_read);
-+
-+/**
-+ * netdev_wait_allrefs - wait until all references are gone.
-+ * @dev: target net_device
-+ *
-+ * This is called when unregistering network devices.
-+ *
-+ * Any protocol or device that holds a reference should register
-+ * for netdevice notification, and cleanup and put back the
-+ * reference if they receive an UNREGISTER event.
-+ * We can get stuck here if buggy protocols don't correctly
-+ * call dev_put.
-+ */
-+static void netdev_wait_allrefs(struct net_device *dev)
-+{
-+ unsigned long rebroadcast_time, warning_time;
-+ int refcnt;
-+
-+ linkwatch_forget_dev(dev);
-+
-+ rebroadcast_time = warning_time = jiffies;
-+ refcnt = netdev_refcnt_read(dev);
-+
-+ while (refcnt != 0) {
-+ if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
-+ rtnl_lock();
-+
-+ /* Rebroadcast unregister notification */
-+ call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
-+
-+ __rtnl_unlock();
-+ rcu_barrier();
-+ rtnl_lock();
-+
-+ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
-+ if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
-+ &dev->state)) {
-+ /* We must not have linkwatch events
-+ * pending on unregister. If this
-+ * happens, we simply run the queue
-+ * unscheduled, resulting in a noop
-+ * for this device.
-+ */
-+ linkwatch_run_queue();
-+ }
-+
-+ __rtnl_unlock();
-+
-+ rebroadcast_time = jiffies;
-+ }
-+
-+ msleep(250);
-+
-+ refcnt = netdev_refcnt_read(dev);
-+
-+ if (time_after(jiffies, warning_time + 10 * HZ)) {
-+ pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
-+ dev->name, refcnt);
-+ warning_time = jiffies;
-+ }
-+ }
-+}
-+
-+/* The sequence is:
-+ *
-+ * rtnl_lock();
-+ * ...
-+ * register_netdevice(x1);
-+ * register_netdevice(x2);
-+ * ...
-+ * unregister_netdevice(y1);
-+ * unregister_netdevice(y2);
-+ * ...
-+ * rtnl_unlock();
-+ * free_netdev(y1);
-+ * free_netdev(y2);
-+ *
-+ * We are invoked by rtnl_unlock().
-+ * This allows us to deal with problems:
-+ * 1) We can delete sysfs objects which invoke hotplug
-+ * without deadlocking with linkwatch via keventd.
-+ * 2) Since we run with the RTNL semaphore not held, we can sleep
-+ * safely in order to wait for the netdev refcnt to drop to zero.
-+ *
-+ * We must not return until all unregister events added during
-+ * the interval the lock was held have been completed.
-+ */
-+void netdev_run_todo(void)
-+{
-+ struct list_head list;
-+
-+ /* Snapshot list, allow later requests */
-+ list_replace_init(&net_todo_list, &list);
-+
-+ __rtnl_unlock();
-+
-+
-+ /* Wait for rcu callbacks to finish before next phase */
-+ if (!list_empty(&list))
-+ rcu_barrier();
-+
-+ while (!list_empty(&list)) {
-+ struct net_device *dev
-+ = list_first_entry(&list, struct net_device, todo_list);
-+ list_del(&dev->todo_list);
-+
-+ rtnl_lock();
-+ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
-+ __rtnl_unlock();
-+
-+ if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
-+ pr_err("network todo '%s' but state %d\n",
-+ dev->name, dev->reg_state);
-+ dump_stack();
-+ continue;
-+ }
-+
-+ dev->reg_state = NETREG_UNREGISTERED;
-+
-+ netdev_wait_allrefs(dev);
-+
-+ /* paranoia */
-+ BUG_ON(netdev_refcnt_read(dev));
-+ BUG_ON(!list_empty(&dev->ptype_all));
-+ BUG_ON(!list_empty(&dev->ptype_specific));
-+ WARN_ON(rcu_access_pointer(dev->ip_ptr));
-+ WARN_ON(rcu_access_pointer(dev->ip6_ptr));
-+ WARN_ON(dev->dn_ptr);
-+
-+ if (dev->destructor)
-+ dev->destructor(dev);
-+
-+ /* Report a network device has been unregistered */
-+ rtnl_lock();
-+ dev_net(dev)->dev_unreg_count--;
-+ __rtnl_unlock();
-+ wake_up(&netdev_unregistering_wq);
-+
-+ /* Free network device */
-+ kobject_put(&dev->dev.kobj);
-+ }
-+}
-+
-+/* Convert net_device_stats to rtnl_link_stats64. They have the same
-+ * fields in the same order, with only the type differing.
-+ */
-+void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
-+ const struct net_device_stats *netdev_stats)
-+{
-+#if BITS_PER_LONG == 64
-+ BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
-+ memcpy(stats64, netdev_stats, sizeof(*stats64));
-+#else
-+ size_t i, n = sizeof(*stats64) / sizeof(u64);
-+ const unsigned long *src = (const unsigned long *)netdev_stats;
-+ u64 *dst = (u64 *)stats64;
-+
-+ BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
-+ sizeof(*stats64) / sizeof(u64));
-+ for (i = 0; i < n; i++)
-+ dst[i] = src[i];
-+#endif
-+}
-+EXPORT_SYMBOL(netdev_stats_to_stats64);
-+
-+/**
-+ * dev_get_stats - get network device statistics
-+ * @dev: device to get statistics from
-+ * @storage: place to store stats
-+ *
-+ * Get network statistics from device. Return @storage.
-+ * The device driver may provide its own method by setting
-+ * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
-+ * otherwise the internal statistics structure is used.
-+ */
-+struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
-+ struct rtnl_link_stats64 *storage)
-+{
-+ const struct net_device_ops *ops = dev->netdev_ops;
-+
-+ if (ops->ndo_get_stats64) {
-+ memset(storage, 0, sizeof(*storage));
-+ ops->ndo_get_stats64(dev, storage);
-+ } else if (ops->ndo_get_stats) {
-+ netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
-+ } else {
-+ netdev_stats_to_stats64(storage, &dev->stats);
-+ }
-+ storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
-+ storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
-+ return storage;
-+}
-+EXPORT_SYMBOL(dev_get_stats);
-+
-+struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
-+{
-+ struct netdev_queue *queue = dev_ingress_queue(dev);
-+
-+#ifdef CONFIG_NET_CLS_ACT
-+ if (queue)
-+ return queue;
-+ queue = kzalloc(sizeof(*queue), GFP_KERNEL);
-+ if (!queue)
-+ return NULL;
-+ netdev_init_one_queue(dev, queue, NULL);
-+ RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
-+ queue->qdisc_sleeping = &noop_qdisc;
-+ rcu_assign_pointer(dev->ingress_queue, queue);
-+#endif
-+ return queue;
-+}
-+
-+static const struct ethtool_ops default_ethtool_ops;
-+
-+void netdev_set_default_ethtool_ops(struct net_device *dev,
-+ const struct ethtool_ops *ops)
-+{
-+ if (dev->ethtool_ops == &default_ethtool_ops)
-+ dev->ethtool_ops = ops;
-+}
-+EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
-+
-+void netdev_freemem(struct net_device *dev)
-+{
-+ char *addr = (char *)dev - dev->padded;
-+
-+ kvfree(addr);
-+}
-+
-+/**
-+ * alloc_netdev_mqs - allocate network device
-+ * @sizeof_priv: size of private data to allocate space for
-+ * @name: device name format string
-+ * @name_assign_type: origin of device name
-+ * @setup: callback to initialize device
-+ * @txqs: the number of TX subqueues to allocate
-+ * @rxqs: the number of RX subqueues to allocate
-+ *
-+ * Allocates a struct net_device with private data area for driver use
-+ * and performs basic initialization. Also allocates subqueue structs
-+ * for each queue on the device.
-+ */
-+struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
-+ unsigned char name_assign_type,
-+ void (*setup)(struct net_device *),
-+ unsigned int txqs, unsigned int rxqs)
-+{
-+ struct net_device *dev;
-+ size_t alloc_size;
-+ struct net_device *p;
-+
-+ BUG_ON(strlen(name) >= sizeof(dev->name));
-+
-+ if (txqs < 1) {
-+ pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
-+ return NULL;
-+ }
-+
-+#ifdef CONFIG_SYSFS
-+ if (rxqs < 1) {
-+ pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
-+ return NULL;
-+ }
-+#endif
-+
-+ alloc_size = sizeof(struct net_device);
-+ if (sizeof_priv) {
-+ /* ensure 32-byte alignment of private area */
-+ alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
-+ alloc_size += sizeof_priv;
-+ }
-+ /* ensure 32-byte alignment of whole construct */
-+ alloc_size += NETDEV_ALIGN - 1;
-+
-+ p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
-+ if (!p)
-+ p = vzalloc(alloc_size);
-+ if (!p)
-+ return NULL;
-+
-+ dev = PTR_ALIGN(p, NETDEV_ALIGN);
-+ dev->padded = (char *)dev - (char *)p;
-+
-+ dev->pcpu_refcnt = alloc_percpu(int);
-+ if (!dev->pcpu_refcnt)
-+ goto free_dev;
-+
-+ if (dev_addr_init(dev))
-+ goto free_pcpu;
-+
-+ dev_mc_init(dev);
-+ dev_uc_init(dev);
-+
-+ dev_net_set(dev, &init_net);
-+
-+ dev->gso_max_size = GSO_MAX_SIZE;
-+ dev->gso_max_segs = GSO_MAX_SEGS;
-+ dev->gso_min_segs = 0;
-+
-+ INIT_LIST_HEAD(&dev->napi_list);
-+ INIT_LIST_HEAD(&dev->unreg_list);
-+ INIT_LIST_HEAD(&dev->close_list);
-+ INIT_LIST_HEAD(&dev->link_watch_list);
-+ INIT_LIST_HEAD(&dev->adj_list.upper);
-+ INIT_LIST_HEAD(&dev->adj_list.lower);
-+ INIT_LIST_HEAD(&dev->all_adj_list.upper);
-+ INIT_LIST_HEAD(&dev->all_adj_list.lower);
-+ INIT_LIST_HEAD(&dev->ptype_all);
-+ INIT_LIST_HEAD(&dev->ptype_specific);
-+ dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
-+ setup(dev);
-+
-+ dev->num_tx_queues = txqs;
-+ dev->real_num_tx_queues = txqs;
-+ if (netif_alloc_netdev_queues(dev))
-+ goto free_all;
-+
-+#ifdef CONFIG_SYSFS
-+ dev->num_rx_queues = rxqs;
-+ dev->real_num_rx_queues = rxqs;
-+ if (netif_alloc_rx_queues(dev))
-+ goto free_all;
-+#endif
-+
-+ strcpy(dev->name, name);
-+ dev->name_assign_type = name_assign_type;
-+ dev->group = INIT_NETDEV_GROUP;
-+ if (!dev->ethtool_ops)
-+ dev->ethtool_ops = &default_ethtool_ops;
-+ return dev;
-+
-+free_all:
-+ free_netdev(dev);
-+ return NULL;
-+
-+free_pcpu:
-+ free_percpu(dev->pcpu_refcnt);
-+free_dev:
-+ netdev_freemem(dev);
-+ return NULL;
-+}
-+EXPORT_SYMBOL(alloc_netdev_mqs);
-+
-+/**
-+ * free_netdev - free network device
-+ * @dev: device
-+ *
-+ * This function does the last stage of destroying an allocated device
-+ * interface. The reference to the device object is released.
-+ * If this is the last reference then it will be freed.
-+ */
-+void free_netdev(struct net_device *dev)
-+{
-+ struct napi_struct *p, *n;
-+
-+ netif_free_tx_queues(dev);
-+#ifdef CONFIG_SYSFS
-+ kvfree(dev->_rx);
-+#endif
-+
-+ kfree(rcu_dereference_protected(dev->ingress_queue, 1));
-+
-+ /* Flush device addresses */
-+ dev_addr_flush(dev);
-+
-+ list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
-+ netif_napi_del(p);
-+
-+ free_percpu(dev->pcpu_refcnt);
-+ dev->pcpu_refcnt = NULL;
-+
-+ /* Compatibility with error handling in drivers */
-+ if (dev->reg_state == NETREG_UNINITIALIZED) {
-+ netdev_freemem(dev);
-+ return;
-+ }
-+
-+ BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
-+ dev->reg_state = NETREG_RELEASED;
-+
-+ /* will free via device release */
-+ put_device(&dev->dev);
-+}
-+EXPORT_SYMBOL(free_netdev);
-+
-+/**
-+ * synchronize_net - Synchronize with packet receive processing
-+ *
-+ * Wait for packets currently being received to be done.
-+ * Does not block later packets from starting.
-+ */
-+void synchronize_net(void)
-+{
-+ might_sleep();
-+ if (rtnl_is_locked())
-+ synchronize_rcu_expedited();
-+ else
-+ synchronize_rcu();
-+}
-+EXPORT_SYMBOL(synchronize_net);
-+
-+/**
-+ * unregister_netdevice_queue - remove device from the kernel
-+ * @dev: device
-+ * @head: list
-+ *
-+ * This function shuts down a device interface and removes it
-+ * from the kernel tables.
-+ * If head not NULL, device is queued to be unregistered later.
-+ *
-+ * Callers must hold the rtnl semaphore. You may want
-+ * unregister_netdev() instead of this.
-+ */
-+
-+void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
-+{
-+ ASSERT_RTNL();
-+
-+ if (head) {
-+ list_move_tail(&dev->unreg_list, head);
-+ } else {
-+ rollback_registered(dev);
-+ /* Finish processing unregister after unlock */
-+ net_set_todo(dev);
-+ }
-+}
-+EXPORT_SYMBOL(unregister_netdevice_queue);
-+
-+/**
-+ * unregister_netdevice_many - unregister many devices
-+ * @head: list of devices
-+ *
-+ * Note: As most callers use a stack allocated list_head,
-+ * we force a list_del() to make sure stack wont be corrupted later.
-+ */
-+void unregister_netdevice_many(struct list_head *head)
-+{
-+ struct net_device *dev;
-+
-+ if (!list_empty(head)) {
-+ rollback_registered_many(head);
-+ list_for_each_entry(dev, head, unreg_list)
-+ net_set_todo(dev);
-+ list_del(head);
-+ }
-+}
-+EXPORT_SYMBOL(unregister_netdevice_many);
-+
-+/**
-+ * unregister_netdev - remove device from the kernel
-+ * @dev: device
-+ *
-+ * This function shuts down a device interface and removes it
-+ * from the kernel tables.
-+ *
-+ * This is just a wrapper for unregister_netdevice that takes
-+ * the rtnl semaphore. In general you want to use this and not
-+ * unregister_netdevice.
-+ */
-+void unregister_netdev(struct net_device *dev)
-+{
-+ rtnl_lock();
-+ unregister_netdevice(dev);
-+ rtnl_unlock();
-+}
-+EXPORT_SYMBOL(unregister_netdev);
-+
-+/**
-+ * dev_change_net_namespace - move device to different nethost namespace
-+ * @dev: device
-+ * @net: network namespace
-+ * @pat: If not NULL name pattern to try if the current device name
-+ * is already taken in the destination network namespace.
-+ *
-+ * This function shuts down a device interface and moves it
-+ * to a new network namespace. On success 0 is returned, on
-+ * a failure a netagive errno code is returned.
-+ *
-+ * Callers must hold the rtnl semaphore.
-+ */
-+
-+int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
-+{
-+ int err;
-+
-+ ASSERT_RTNL();
-+
-+ /* Don't allow namespace local devices to be moved. */
-+ err = -EINVAL;
-+ if (dev->features & NETIF_F_NETNS_LOCAL)
-+ goto out;
-+
-+ /* Ensure the device has been registrered */
-+ if (dev->reg_state != NETREG_REGISTERED)
-+ goto out;
-+
-+ /* Get out if there is nothing todo */
-+ err = 0;
-+ if (net_eq(dev_net(dev), net))
-+ goto out;
-+
-+ /* Pick the destination device name, and ensure
-+ * we can use it in the destination network namespace.
-+ */
-+ err = -EEXIST;
-+ if (__dev_get_by_name(net, dev->name)) {
-+ /* We get here if we can't use the current device name */
-+ if (!pat)
-+ goto out;
-+ if (dev_get_valid_name(net, dev, pat) < 0)
-+ goto out;
-+ }
-+
-+ /*
-+ * And now a mini version of register_netdevice unregister_netdevice.
-+ */
-+
-+ /* If device is running close it first. */
-+ dev_close(dev);
-+
-+ /* And unlink it from device chain */
-+ err = -ENODEV;
-+ unlist_netdevice(dev);
-+
-+ synchronize_net();
-+
-+ /* Shutdown queueing discipline. */
-+ dev_shutdown(dev);
-+
-+ /* Notify protocols, that we are about to destroy
-+ this device. They should clean all the things.
-+
-+ Note that dev->reg_state stays at NETREG_REGISTERED.
-+ This is wanted because this way 8021q and macvlan know
-+ the device is just moving and can keep their slaves up.
-+ */
-+ call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
-+ rcu_barrier();
-+ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
-+ rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
-+
-+ /*
-+ * Flush the unicast and multicast chains
-+ */
-+ dev_uc_flush(dev);
-+ dev_mc_flush(dev);
-+
-+ /* Send a netdev-removed uevent to the old namespace */
-+ kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
-+ netdev_adjacent_del_links(dev);
-+
-+ /* Actually switch the network namespace */
-+ dev_net_set(dev, net);
-+
-+ /* If there is an ifindex conflict assign a new one */
-+ if (__dev_get_by_index(net, dev->ifindex))
-+ dev->ifindex = dev_new_index(net);
-+
-+ /* Send a netdev-add uevent to the new namespace */
-+ kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
-+ netdev_adjacent_add_links(dev);
-+
-+ /* Fixup kobjects */
-+ err = device_rename(&dev->dev, dev->name);
-+ WARN_ON(err);
-+
-+ /* Add the device back in the hashes */
-+ list_netdevice(dev);
-+
-+ /* Notify protocols, that a new device appeared. */
-+ call_netdevice_notifiers(NETDEV_REGISTER, dev);
-+
-+ /*
-+ * Prevent userspace races by waiting until the network
-+ * device is fully setup before sending notifications.
-+ */
-+ rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
-+
-+ synchronize_net();
-+ err = 0;
-+out:
-+ return err;
-+}
-+EXPORT_SYMBOL_GPL(dev_change_net_namespace);
-+
-+static int dev_cpu_callback(struct notifier_block *nfb,
-+ unsigned long action,
-+ void *ocpu)
-+{
-+ struct sk_buff **list_skb;
-+ struct sk_buff *skb;
-+ unsigned int cpu, oldcpu = (unsigned long)ocpu;
-+ struct softnet_data *sd, *oldsd;
-+
-+ if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
-+ return NOTIFY_OK;
-+
-+ local_irq_disable();
-+ cpu = smp_processor_id();
-+ sd = &per_cpu(softnet_data, cpu);
-+ oldsd = &per_cpu(softnet_data, oldcpu);
-+
-+ /* Find end of our completion_queue. */
-+ list_skb = &sd->completion_queue;
-+ while (*list_skb)
-+ list_skb = &(*list_skb)->next;
-+ /* Append completion queue from offline CPU. */
-+ *list_skb = oldsd->completion_queue;
-+ oldsd->completion_queue = NULL;
-+
-+ /* Append output queue from offline CPU. */
-+ if (oldsd->output_queue) {
-+ *sd->output_queue_tailp = oldsd->output_queue;
-+ sd->output_queue_tailp = oldsd->output_queue_tailp;
-+ oldsd->output_queue = NULL;
-+ oldsd->output_queue_tailp = &oldsd->output_queue;
-+ }
-+ /* Append NAPI poll list from offline CPU, with one exception :
-+ * process_backlog() must be called by cpu owning percpu backlog.
-+ * We properly handle process_queue & input_pkt_queue later.
-+ */
-+ while (!list_empty(&oldsd->poll_list)) {
-+ struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
-+ struct napi_struct,
-+ poll_list);
-+
-+ list_del_init(&napi->poll_list);
-+ if (napi->poll == process_backlog)
-+ napi->state = 0;
-+ else
-+ ____napi_schedule(sd, napi);
-+ }
-+
-+ raise_softirq_irqoff(NET_TX_SOFTIRQ);
-+ local_irq_enable();
-+
-+ /* Process offline CPU's input_pkt_queue */
-+ while ((skb = __skb_dequeue(&oldsd->process_queue))) {
-+ netif_rx_ni(skb);
-+ input_queue_head_incr(oldsd);
-+ }
-+ while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
-+ netif_rx_ni(skb);
-+ input_queue_head_incr(oldsd);
-+ }
-+
-+ return NOTIFY_OK;
-+}
-+
-+
-+/**
-+ * netdev_increment_features - increment feature set by one
-+ * @all: current feature set
-+ * @one: new feature set
-+ * @mask: mask feature set
-+ *
-+ * Computes a new feature set after adding a device with feature set
-+ * @one to the master device with current feature set @all. Will not
-+ * enable anything that is off in @mask. Returns the new feature set.
-+ */
-+netdev_features_t netdev_increment_features(netdev_features_t all,
-+ netdev_features_t one, netdev_features_t mask)
-+{
-+ if (mask & NETIF_F_GEN_CSUM)
-+ mask |= NETIF_F_ALL_CSUM;
-+ mask |= NETIF_F_VLAN_CHALLENGED;
-+
-+ all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
-+ all &= one | ~NETIF_F_ALL_FOR_ALL;
-+
-+ /* If one device supports hw checksumming, set for all. */
-+ if (all & NETIF_F_GEN_CSUM)
-+ all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
-+
-+ return all;
-+}
-+EXPORT_SYMBOL(netdev_increment_features);
-+
-+static struct hlist_head * __net_init netdev_create_hash(void)
-+{
-+ int i;
-+ struct hlist_head *hash;
-+
-+ hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
-+ if (hash != NULL)
-+ for (i = 0; i < NETDEV_HASHENTRIES; i++)
-+ INIT_HLIST_HEAD(&hash[i]);
-+
-+ return hash;
-+}
-+
-+/* Initialize per network namespace state */
-+static int __net_init netdev_init(struct net *net)
-+{
-+ if (net != &init_net)
-+ INIT_LIST_HEAD(&net->dev_base_head);
-+
-+ net->dev_name_head = netdev_create_hash();
-+ if (net->dev_name_head == NULL)
-+ goto err_name;
-+
-+ net->dev_index_head = netdev_create_hash();
-+ if (net->dev_index_head == NULL)
-+ goto err_idx;
-+
-+ return 0;
-+
-+err_idx:
-+ kfree(net->dev_name_head);
-+err_name:
-+ return -ENOMEM;
-+}
-+
-+/**
-+ * netdev_drivername - network driver for the device
-+ * @dev: network device
-+ *
-+ * Determine network driver for device.
-+ */
-+const char *netdev_drivername(const struct net_device *dev)
-+{
-+ const struct device_driver *driver;
-+ const struct device *parent;
-+ const char *empty = "";
-+
-+ parent = dev->dev.parent;
-+ if (!parent)
-+ return empty;
-+
-+ driver = parent->driver;
-+ if (driver && driver->name)
-+ return driver->name;
-+ return empty;
-+}
-+
-+static void __netdev_printk(const char *level, const struct net_device *dev,
-+ struct va_format *vaf)
-+{
-+ if (dev && dev->dev.parent) {
-+ dev_printk_emit(level[1] - '0',
-+ dev->dev.parent,
-+ "%s %s %s%s: %pV",
-+ dev_driver_string(dev->dev.parent),
-+ dev_name(dev->dev.parent),
-+ netdev_name(dev), netdev_reg_state(dev),
-+ vaf);
-+ } else if (dev) {
-+ printk("%s%s%s: %pV",
-+ level, netdev_name(dev), netdev_reg_state(dev), vaf);
-+ } else {
-+ printk("%s(NULL net_device): %pV", level, vaf);
-+ }
-+}
-+
-+void netdev_printk(const char *level, const struct net_device *dev,
-+ const char *format, ...)
-+{
-+ struct va_format vaf;
-+ va_list args;
-+
-+ va_start(args, format);
-+
-+ vaf.fmt = format;
-+ vaf.va = &args;
-+
-+ __netdev_printk(level, dev, &vaf);
-+
-+ va_end(args);
-+}
-+EXPORT_SYMBOL(netdev_printk);
-+
-+#define define_netdev_printk_level(func, level) \
-+void func(const struct net_device *dev, const char *fmt, ...) \
-+{ \
-+ struct va_format vaf; \
-+ va_list args; \
-+ \
-+ va_start(args, fmt); \
-+ \
-+ vaf.fmt = fmt; \
-+ vaf.va = &args; \
-+ \
-+ __netdev_printk(level, dev, &vaf); \
-+ \
-+ va_end(args); \
-+} \
-+EXPORT_SYMBOL(func);
-+
-+define_netdev_printk_level(netdev_emerg, KERN_EMERG);
-+define_netdev_printk_level(netdev_alert, KERN_ALERT);
-+define_netdev_printk_level(netdev_crit, KERN_CRIT);
-+define_netdev_printk_level(netdev_err, KERN_ERR);
-+define_netdev_printk_level(netdev_warn, KERN_WARNING);
-+define_netdev_printk_level(netdev_notice, KERN_NOTICE);
-+define_netdev_printk_level(netdev_info, KERN_INFO);
-+
-+static void __net_exit netdev_exit(struct net *net)
-+{
-+ kfree(net->dev_name_head);
-+ kfree(net->dev_index_head);
-+}
-+
-+static struct pernet_operations __net_initdata netdev_net_ops = {
-+ .init = netdev_init,
-+ .exit = netdev_exit,
-+};
-+
-+static void __net_exit default_device_exit(struct net *net)
-+{
-+ struct net_device *dev, *aux;
-+ /*
-+ * Push all migratable network devices back to the
-+ * initial network namespace
-+ */
-+ rtnl_lock();
-+ for_each_netdev_safe(net, dev, aux) {
-+ int err;
-+ char fb_name[IFNAMSIZ];
-+
-+ /* Ignore unmoveable devices (i.e. loopback) */
-+ if (dev->features & NETIF_F_NETNS_LOCAL)
-+ continue;
-+
-+ /* Leave virtual devices for the generic cleanup */
-+ if (dev->rtnl_link_ops)
-+ continue;
-+
-+ /* Push remaining network devices to init_net */
-+ snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
-+ err = dev_change_net_namespace(dev, &init_net, fb_name);
-+ if (err) {
-+ pr_emerg("%s: failed to move %s to init_net: %d\n",
-+ __func__, dev->name, err);
-+ BUG();
-+ }
-+ }
-+ rtnl_unlock();
-+}
-+
-+static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
-+{
-+ /* Return with the rtnl_lock held when there are no network
-+ * devices unregistering in any network namespace in net_list.
-+ */
-+ struct net *net;
-+ bool unregistering;
-+ DEFINE_WAIT_FUNC(wait, woken_wake_function);
-+
-+ add_wait_queue(&netdev_unregistering_wq, &wait);
-+ for (;;) {
-+ unregistering = false;
-+ rtnl_lock();
-+ list_for_each_entry(net, net_list, exit_list) {
-+ if (net->dev_unreg_count > 0) {
-+ unregistering = true;
-+ break;
-+ }
-+ }
-+ if (!unregistering)
-+ break;
-+ __rtnl_unlock();
-+
-+ wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
-+ }
-+ remove_wait_queue(&netdev_unregistering_wq, &wait);
-+}
-+
-+static void __net_exit default_device_exit_batch(struct list_head *net_list)
-+{
-+ /* At exit all network devices most be removed from a network
-+ * namespace. Do this in the reverse order of registration.
-+ * Do this across as many network namespaces as possible to
-+ * improve batching efficiency.
-+ */
-+ struct net_device *dev;
-+ struct net *net;
-+ LIST_HEAD(dev_kill_list);
-+
-+ /* To prevent network device cleanup code from dereferencing
-+ * loopback devices or network devices that have been freed
-+ * wait here for all pending unregistrations to complete,
-+ * before unregistring the loopback device and allowing the
-+ * network namespace be freed.
-+ *
-+ * The netdev todo list containing all network devices
-+ * unregistrations that happen in default_device_exit_batch
-+ * will run in the rtnl_unlock() at the end of
-+ * default_device_exit_batch.
-+ */
-+ rtnl_lock_unregistering(net_list);
-+ list_for_each_entry(net, net_list, exit_list) {
-+ for_each_netdev_reverse(net, dev) {
-+ if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
-+ dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
-+ else
-+ unregister_netdevice_queue(dev, &dev_kill_list);
-+ }
-+ }
-+ unregister_netdevice_many(&dev_kill_list);
-+ rtnl_unlock();
-+}
-+
-+static struct pernet_operations __net_initdata default_device_ops = {
-+ .exit = default_device_exit,
-+ .exit_batch = default_device_exit_batch,
-+};
-+
-+/*
-+ * Initialize the DEV module. At boot time this walks the device list and
-+ * unhooks any devices that fail to initialise (normally hardware not
-+ * present) and leaves us with a valid list of present and active devices.
-+ *
-+ */
-+
-+/*
-+ * This is called single threaded during boot, so no need
-+ * to take the rtnl semaphore.
-+ */
-+static int __init net_dev_init(void)
-+{
-+ int i, rc = -ENOMEM;
-+
-+ BUG_ON(!dev_boot_phase);
-+
-+ if (dev_proc_init())
-+ goto out;
-+
-+ if (netdev_kobject_init())
-+ goto out;
-+
-+ INIT_LIST_HEAD(&ptype_all);
-+ for (i = 0; i < PTYPE_HASH_SIZE; i++)
-+ INIT_LIST_HEAD(&ptype_base[i]);
-+
-+ INIT_LIST_HEAD(&offload_base);
-+
-+ if (register_pernet_subsys(&netdev_net_ops))
-+ goto out;
-+
-+ /*
-+ * Initialise the packet receive queues.
-+ */
-+
-+ for_each_possible_cpu(i) {
-+ struct softnet_data *sd = &per_cpu(softnet_data, i);
-+
-+ skb_queue_head_init(&sd->input_pkt_queue);
-+ skb_queue_head_init(&sd->process_queue);
-+ INIT_LIST_HEAD(&sd->poll_list);
-+ sd->output_queue_tailp = &sd->output_queue;
-+#ifdef CONFIG_RPS
-+ sd->csd.func = rps_trigger_softirq;
-+ sd->csd.info = sd;
-+ sd->cpu = i;
-+#endif
-+
-+ sd->backlog.poll = process_backlog;
-+ sd->backlog.weight = weight_p;
-+ }
-+
-+ dev_boot_phase = 0;
-+
-+ /* The loopback device is special if any other network devices
-+ * is present in a network namespace the loopback device must
-+ * be present. Since we now dynamically allocate and free the
-+ * loopback device ensure this invariant is maintained by
-+ * keeping the loopback device as the first device on the
-+ * list of network devices. Ensuring the loopback devices
-+ * is the first device that appears and the last network device
-+ * that disappears.
-+ */
-+ if (register_pernet_device(&loopback_net_ops))
-+ goto out;
-+
-+ if (register_pernet_device(&default_device_ops))
-+ goto out;
-+
-+ open_softirq(NET_TX_SOFTIRQ, net_tx_action);
-+ open_softirq(NET_RX_SOFTIRQ, net_rx_action);
-+
-+ hotcpu_notifier(dev_cpu_callback, 0);
-+ dst_init();
-+ rc = 0;
-+out:
-+ return rc;
-+}
-+
-+subsys_initcall(net_dev_init);
diff -Nur linux-4.1.10.orig/net/core/skbuff.c linux-4.1.10/net/core/skbuff.c
--- linux-4.1.10.orig/net/core/skbuff.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/core/skbuff.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/core/skbuff.c 2015-10-12 22:33:32.344672261 +0200
@@ -63,6 +63,7 @@
#include <linux/errqueue.h>
#include <linux/prefetch.h>
@@ -70085,7 +26772,7 @@ diff -Nur linux-4.1.10.orig/net/core/skbuff.c linux-4.1.10/net/core/skbuff.c
diff -Nur linux-4.1.10.orig/net/core/sock.c linux-4.1.10/net/core/sock.c
--- linux-4.1.10.orig/net/core/sock.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/core/sock.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/core/sock.c 2015-10-12 22:33:32.344672261 +0200
@@ -2370,12 +2370,11 @@
if (sk->sk_lock.owned)
__lock_sock(sk);
@@ -70102,7 +26789,7 @@ diff -Nur linux-4.1.10.orig/net/core/sock.c linux-4.1.10/net/core/sock.c
diff -Nur linux-4.1.10.orig/net/ipv4/icmp.c linux-4.1.10/net/ipv4/icmp.c
--- linux-4.1.10.orig/net/ipv4/icmp.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/ipv4/icmp.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/ipv4/icmp.c 2015-10-12 22:33:32.344672261 +0200
@@ -69,6 +69,7 @@
#include <linux/jiffies.h>
#include <linux/kernel.h>
@@ -70156,7 +26843,7 @@ diff -Nur linux-4.1.10.orig/net/ipv4/icmp.c linux-4.1.10/net/ipv4/icmp.c
return true;
diff -Nur linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.10/net/ipv4/sysctl_net_ipv4.c
--- linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/ipv4/sysctl_net_ipv4.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/ipv4/sysctl_net_ipv4.c 2015-10-12 22:33:32.344672261 +0200
@@ -779,6 +779,13 @@
.proc_handler = proc_dointvec
},
@@ -70173,7 +26860,7 @@ diff -Nur linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.10/net/ipv4/sys
.maxlen = sizeof(int),
diff -Nur linux-4.1.10.orig/net/mac80211/rx.c linux-4.1.10/net/mac80211/rx.c
--- linux-4.1.10.orig/net/mac80211/rx.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/mac80211/rx.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/mac80211/rx.c 2015-10-12 22:33:32.344672261 +0200
@@ -3554,7 +3554,7 @@
struct ieee80211_supported_band *sband;
struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
@@ -70185,7 +26872,7 @@ diff -Nur linux-4.1.10.orig/net/mac80211/rx.c linux-4.1.10/net/mac80211/rx.c
goto drop;
diff -Nur linux-4.1.10.orig/net/netfilter/core.c linux-4.1.10/net/netfilter/core.c
--- linux-4.1.10.orig/net/netfilter/core.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/netfilter/core.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/netfilter/core.c 2015-10-12 22:33:32.344672261 +0200
@@ -22,11 +22,17 @@
#include <linux/proc_fs.h>
#include <linux/mutex.h>
@@ -70206,7 +26893,7 @@ diff -Nur linux-4.1.10.orig/net/netfilter/core.c linux-4.1.10/net/netfilter/core
const struct nf_afinfo __rcu *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly;
diff -Nur linux-4.1.10.orig/net/packet/af_packet.c linux-4.1.10/net/packet/af_packet.c
--- linux-4.1.10.orig/net/packet/af_packet.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/packet/af_packet.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/packet/af_packet.c 2015-10-12 22:33:32.344672261 +0200
@@ -63,6 +63,7 @@
#include <linux/if_packet.h>
#include <linux/wireless.h>
@@ -70235,7 +26922,7 @@ diff -Nur linux-4.1.10.orig/net/packet/af_packet.c linux-4.1.10/net/packet/af_pa
prb_close_block(pkc, pbd, po, status);
diff -Nur linux-4.1.10.orig/net/rds/ib_rdma.c linux-4.1.10/net/rds/ib_rdma.c
--- linux-4.1.10.orig/net/rds/ib_rdma.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/rds/ib_rdma.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/rds/ib_rdma.c 2015-10-12 22:33:32.344672261 +0200
@@ -34,6 +34,7 @@
#include <linux/slab.h>
#include <linux/rculist.h>
@@ -70255,7 +26942,7 @@ diff -Nur linux-4.1.10.orig/net/rds/ib_rdma.c linux-4.1.10/net/rds/ib_rdma.c
diff -Nur linux-4.1.10.orig/net/sched/sch_generic.c linux-4.1.10/net/sched/sch_generic.c
--- linux-4.1.10.orig/net/sched/sch_generic.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/sched/sch_generic.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/sched/sch_generic.c 2015-10-12 22:33:32.344672261 +0200
@@ -894,7 +894,7 @@
/* Wait for outstanding qdisc_run calls. */
list_for_each_entry(dev, head, close_list)
@@ -70267,7 +26954,7 @@ diff -Nur linux-4.1.10.orig/net/sched/sch_generic.c linux-4.1.10/net/sched/sch_g
void dev_deactivate(struct net_device *dev)
diff -Nur linux-4.1.10.orig/net/sunrpc/svc_xprt.c linux-4.1.10/net/sunrpc/svc_xprt.c
--- linux-4.1.10.orig/net/sunrpc/svc_xprt.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/net/sunrpc/svc_xprt.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/net/sunrpc/svc_xprt.c 2015-10-12 22:33:32.344672261 +0200
@@ -341,7 +341,7 @@
goto out;
}
@@ -70297,7 +26984,7 @@ diff -Nur linux-4.1.10.orig/net/sunrpc/svc_xprt.c linux-4.1.10/net/sunrpc/svc_xp
}
diff -Nur linux-4.1.10.orig/scripts/mkcompile_h linux-4.1.10/scripts/mkcompile_h
--- linux-4.1.10.orig/scripts/mkcompile_h 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/scripts/mkcompile_h 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/scripts/mkcompile_h 2015-10-12 22:33:32.344672261 +0200
@@ -4,7 +4,8 @@
ARCH=$2
SMP=$3
@@ -70318,7 +27005,7 @@ diff -Nur linux-4.1.10.orig/scripts/mkcompile_h linux-4.1.10/scripts/mkcompile_h
# Truncate to maximum length
diff -Nur linux-4.1.10.orig/sound/core/pcm_native.c linux-4.1.10/sound/core/pcm_native.c
--- linux-4.1.10.orig/sound/core/pcm_native.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/sound/core/pcm_native.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/sound/core/pcm_native.c 2015-10-12 22:33:32.348671997 +0200
@@ -123,7 +123,7 @@
void snd_pcm_stream_lock_irq(struct snd_pcm_substream *substream)
{
@@ -70357,7 +27044,7 @@ diff -Nur linux-4.1.10.orig/sound/core/pcm_native.c linux-4.1.10/sound/core/pcm_
diff -Nur linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.10/sound/soc/intel/atom/sst/sst.c
--- linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/sound/soc/intel/atom/sst/sst.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/sound/soc/intel/atom/sst/sst.c 2015-10-12 22:33:32.348671997 +0200
@@ -368,8 +368,8 @@
* initialize by FW or driver when firmware is loaded
*/
@@ -70371,7 +27058,7 @@ diff -Nur linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.10/sound/so
diff -Nur linux-4.1.10.orig/virt/kvm/async_pf.c linux-4.1.10/virt/kvm/async_pf.c
--- linux-4.1.10.orig/virt/kvm/async_pf.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/virt/kvm/async_pf.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/virt/kvm/async_pf.c 2015-10-12 22:33:32.348671997 +0200
@@ -94,8 +94,8 @@
trace_kvm_async_pf_completed(addr, gva);
@@ -70385,7 +27072,7 @@ diff -Nur linux-4.1.10.orig/virt/kvm/async_pf.c linux-4.1.10/virt/kvm/async_pf.c
kvm_put_kvm(vcpu->kvm);
diff -Nur linux-4.1.10.orig/virt/kvm/kvm_main.c linux-4.1.10/virt/kvm/kvm_main.c
--- linux-4.1.10.orig/virt/kvm/kvm_main.c 2015-10-03 13:49:38.000000000 +0200
-+++ linux-4.1.10/virt/kvm/kvm_main.c 2015-10-07 18:00:08.000000000 +0200
++++ linux-4.1.10/virt/kvm/kvm_main.c 2015-10-12 22:33:32.348671997 +0200
@@ -218,7 +218,7 @@
vcpu->kvm = kvm;
vcpu->vcpu_id = id;
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-13 04:43:28 +0000