prlimit: Translate old_rlimit from RLIM64_INFINITY to RLIM_INFINITY [BZ #22678]

prlimit called without a new value fails on 32-bit machines if any of
the soft or hard limits are infinity. This is because prlimit does not
translate old_rlimit from RLIM64_INFINITY to RLIM_INFINITY, but checks
that the value returned by the prlimit64 syscall fits into a 32-bit
value, like it is done for example in getrlimit. Note that on the
other hand new_rlimit is correctly translated from RLIM_INFINITY to
RLIM64_INFINITY before calling the syscall.

This patch fixes that.

Changelog:
	[BZ #22678]
	* sysdeps/unix/sysv/linux/prlimit.c (prlimit): Translate
	old_rlimit from RLIM64_INFINITY to RLIM_INFINITY.

diff --git a/ChangeLog b/ChangeLog
index fd0fc0bc71a38b828365662f629d6fef6aef3497..53c3d62b2ef65bcbb38361650e443f98bd753acf 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2018-01-05  Aurelien Jarno  <aurelien@aurel32.net>
+
+       [BZ #22678]
+       * sysdeps/unix/sysv/linux/prlimit.c (prlimit): Translate
+       old_rlimit from RLIM64_INFINITY to RLIM_INFINITY.
+
 2018-01-05  Aurelien Jarno  <aurelien@aurel32.net>
            Adhemerval Zanella  <adhemerval.zanella@linaro.org>
 

diff --git a/sysdeps/unix/sysv/linux/prlimit.c b/sysdeps/unix/sysv/linux/prlimit.c
index 9db8e821b34c1a1552419df5e42304d58d8957b6..2fa0642c766bab4f16e79ab7e53133fd14f69fb1 100644 (file)
--- a/sysdeps/unix/sysv/linux/prlimit.c
+++ b/sysdeps/unix/sysv/linux/prlimit.c
@@ -50,21 +50,24 @@ prlimit (__pid_t pid, enum __rlimit_resource resource,
     {
       /* The prlimit64 syscall is ill-designed for 32-bit machines.
         We have to provide a 32-bit variant since otherwise the LFS
-        system would not work.  But what shall we do if the syscall
-        succeeds but the old values do not fit into a rlimit
-        structure?  We cannot return an error because the operation
-        itself worked.  Best is perhaps to return RLIM_INFINITY.  */
+        system would not work.  The infinity value can be translated,
+        but otherwise what shall we do if the syscall succeeds but the
+        old values do not fit into a rlimit structure?  We cannot return
+        an error because the operation itself worked.  Best is perhaps
+        to return RLIM_INFINITY.  */
       old_rlimit->rlim_cur = old_rlimit64_mem.rlim_cur;
       if (old_rlimit->rlim_cur != old_rlimit64_mem.rlim_cur)
        {
-         if (new_rlimit == NULL)
+         if ((new_rlimit == NULL)
+             && (old_rlimit64_mem.rlim_cur != RLIM64_INFINITY))
            return INLINE_SYSCALL_ERROR_RETURN_VALUE (EOVERFLOW);
          old_rlimit->rlim_cur = RLIM_INFINITY;
        }
       old_rlimit->rlim_max = old_rlimit64_mem.rlim_max;
       if (old_rlimit->rlim_max != old_rlimit64_mem.rlim_max)
        {
-         if (new_rlimit == NULL)
+         if ((new_rlimit == NULL)
+             && (old_rlimit64_mem.rlim_max != RLIM64_INFINITY))
            return INLINE_SYSCALL_ERROR_RETURN_VALUE (EOVERFLOW);
          old_rlimit->rlim_max = RLIM_INFINITY;
        }