setrlimit/getrlimit: fix prlimit64 syscall use for 32-bit CPUs

Commit 95e38b37 ("add support for systems without legacy setrlimit/getrlimit
syscalls") has added use of the prlimit64 syscall in getrlimit and setrlimit
functions. This change causes memory corruption on getrlimit call for 32-bit
CPUs like ARC, as ARC doesn't have ugetrlimit syscall and uses prlimit64.
Also, setrlimit has been broken by prlimit64 call on 32-bit CPUs like, i386,
ARM, ARC.

For the prlimit64 syscall the kernel expects an rlimit struct with 64-bit fields,
but on 32-bit CPUs without _FILE_OFFSET_BITS=64 the struct rlimit has 32-bit
fields.

Add safe implementations of getrlimit, setrlimit, prlimit for 32-bit CPUs with a
local struct rlimit64 variable for use in the prlimit64 syscall.
For 64-bit CPUs and configurations with _FILE_OFFSET_BITS=64 use
getrlimit, setrlimit, prlimit as aliases to getrlimit64, setrlimit64 and
prlimit64. Add a new function prlimit64.

Tested on aarch64, arm, i386, arc.

Fixes: 95e38b37 ("add support for systems without legacy setrlimit/getrlimit syscalls")
Signed-off-by: Pavel Kozlov <pavel.kozlov@synopsys.com>

1 files changed, 48 insertions, 5 deletions

diff --git a/libc/sysdeps/linux/common/prlimit.c b/libc/sysdeps/linux/common/prlimit.c
index f44dc1664..f59ade3a3 100644
--- a/libc/sysdeps/linux/common/prlimit.c
+++ b/libc/sysdeps/linux/common/prlimit.c
@@ -17,14 +17,57 @@
 
 #include <sys/resource.h>
 #include <sysdep.h>
-#include <bits/kernel-features.h>
+#include <stddef.h> // needed for NULL to be defined
 
-#if defined __ASSUME_PRLIMIT64
+#if defined(__NR_prlimit64) && __WORDSIZE == 32 && !defined(__USE_FILE_OFFSET64)
 int
 prlimit (__pid_t pid, enum __rlimit_resource resource,
-	     const struct rlimit *new_rlimit, struct rlimit *old_rlimit)
+	 const struct rlimit *new_rlimit, struct rlimit *old_rlimit)
 {
-  return INLINE_SYSCALL (prlimit64, 4, pid, resource, new_rlimit,
-			      old_rlimit);
+	struct rlimit64 new_rlimit64;
+	struct rlimit64 old_rlimit64;
+	int res;
+
+	if (new_rlimit != NULL) {
+		if (new_rlimit->rlim_cur == RLIM_INFINITY)
+			new_rlimit64.rlim_cur = RLIM64_INFINITY;
+		else
+			new_rlimit64.rlim_cur = new_rlimit->rlim_cur;
+		if (new_rlimit->rlim_max == RLIM_INFINITY)
+			new_rlimit64.rlim_max = RLIM64_INFINITY;
+		else
+			new_rlimit64.rlim_max = new_rlimit->rlim_max;
+	}
+
+	res = INLINE_SYSCALL (prlimit64, 4, pid, resource, &new_rlimit64,
+			      &old_rlimit64);
+
+	if (res == 0 && old_rlimit != NULL) {
+		/* If the syscall succeeds but the values do not fit into a
+		   rlimit structure set EOVERFLOW errno and retrun -1.
+		   With current Linux implementation of the prlimit64 syscall,
+		   overflow can't happen. An extra condition has been added to get 
+		   the same behavior as in glibc for future potential overflows. */
+		old_rlimit->rlim_cur = old_rlimit64.rlim_cur;
+		if (old_rlimit64.rlim_cur != old_rlimit->rlim_cur) {
+			if (new_rlimit == NULL && 
+			    old_rlimit64.rlim_cur != RLIM64_INFINITY) {
+				__set_errno(EOVERFLOW);
+				return -1;
+			}
+			old_rlimit->rlim_cur = RLIM_INFINITY;
+		}
+		old_rlimit->rlim_max = old_rlimit64.rlim_max;
+		if (old_rlimit64.rlim_max != old_rlimit->rlim_max) {
+			if (new_rlimit == NULL &&
+			    old_rlimit64.rlim_max != RLIM64_INFINITY) {
+				__set_errno(EOVERFLOW);
+				return -1;
+			}
+			old_rlimit->rlim_cur = RLIM_INFINITY;
+		}
+	}
+
+	return res;
 }
 #endif