Fix #10968: spawnProcess fails when RLIMIT_NOFILE exceeds int.max

std/process.d

@@ -1046,21 +1046,16 @@ private Pid spawnProcessPosix(scope const(char[])[] args,
                         if (getrlimit(RLIMIT_NOFILE, &r) != 0)
                             abortOnError(forkPipeOut, InternalError.getrlimit, .errno);
 
-                        immutable maxDescriptors = cast(int) r.rlim_cur;
+                        immutable maxDescriptors = r.rlim_cur;
 
                         // Missing druntime declaration
                         pragma(mangle, "dirfd")
                         extern(C) nothrow @nogc int dirfd(DIR* dir);
 
-                        DIR* dir = null;
-
-                        // We read from /dev/fd or /proc/self/fd only if the limit is high enough
-                        if (maxDescriptors > 128*1024)
-                        {
-                            // Try to open the directory /dev/fd or /proc/self/fd
-                            dir = opendir("/dev/fd");
-                            if (dir is null) dir = opendir("/proc/self/fd");
-                        }
+                        // Always try /dev/fd enumeration first — it's the most
+                        // efficient approach and handles unlimited RLIMIT_NOFILE.
+                        DIR* dir = opendir("/dev/fd");
+                        if (dir is null) dir = opendir("/proc/self/fd");
 
                         // If we have a directory, close all file descriptors except stdin, stdout, and stderr
                         if (dir)
@@ -1087,17 +1082,17 @@ private Pid spawnProcessPosix(scope const(char[])[] args,
                         }
                         else
                         {
-                            // This is going to allocate 8 bytes for each possible file descriptor from lowfd to r.rlim_cur
                             if (maxDescriptors <= 128*1024)
                             {
+                                // This is going to allocate 8 bytes for each possible file descriptor from lowfd to rlim_cur.
                                 // NOTE: malloc() and getrlimit() are not on the POSIX async
                                 // signal safe functions list, but practically this should
                                 // not be a problem. Java VM and CPython also use malloc()
                                 // in its own implementation via opendir().
                                 import core.stdc.stdlib : malloc;
                                 import core.sys.posix.poll : pollfd, poll, POLLNVAL;
 
-                                immutable maxToClose = maxDescriptors - lowfd;
+                                immutable maxToClose = cast(int)(maxDescriptors - lowfd);
 
                                 // Call poll() to see which ones are actually open:
                                 auto pfds = cast(pollfd*) malloc(pollfd.sizeof * maxToClose);
@@ -1126,8 +1121,12 @@ private Pid spawnProcessPosix(scope const(char[])[] args,
                             else
                             {
                             LslowClose:
-                                // Fall back to closing everything.
-                                foreach (i; lowfd .. maxDescriptors)
+                                // Fall back to closing everything up to a sane limit.
+                                // When rlim_cur is huge (e.g. unlimited), cap to avoid
+                                // iterating over billions of file descriptors.
+                                immutable closeMax = cast(int)
+                                    (maxDescriptors > 1_048_576 ? 1_048_576 : maxDescriptors);
+                                foreach (i; lowfd .. closeMax)
                                 {
                                     close(i);
                                 }
@@ -1800,6 +1799,44 @@ version (Posix) @system unittest
     testFDs();
 }
 
+// Test that spawning a process works when RLIMIT_NOFILE is very large.
+// Regression test: a cast(int) of rlim_cur caused overflow when the limit
+// was unlimited (RLIM_INFINITY), making the fd-closing code attempt a
+// massive malloc that would fail with "Cannot allocate memory".
+version (Posix) @system unittest
+{
+    import core.sys.posix.sys.resource : rlimit, rlim_t, getrlimit, setrlimit, RLIMIT_NOFILE;
+
+    // This test only applies on platforms where rlim_t can exceed int.max.
+    static if (rlim_t.sizeof > int.sizeof)
+    {
+        // Save current limit
+        rlimit originalLimit;
+        if (getrlimit(RLIMIT_NOFILE, &originalLimit) != 0)
+            return; // Can't test if we can't get the limit
+
+        // Set RLIMIT_NOFILE to a value that overflows int (> int.max)
+        rlimit highLimit;
+        highLimit.rlim_cur = cast(rlim_t) int.max + 1;
+        highLimit.rlim_max = originalLimit.rlim_max;
+
+        // If we can't raise the limit (e.g. no permission), try with rlim_max
+        if (setrlimit(RLIMIT_NOFILE, &highLimit) != 0)
+        {
+            highLimit.rlim_cur = originalLimit.rlim_max;
+            if (highLimit.rlim_cur <= int.max)
+                return; // Can't set a high enough limit to test the overflow
+            if (setrlimit(RLIMIT_NOFILE, &highLimit) != 0)
+                return;
+        }
+        scope(exit) setrlimit(RLIMIT_NOFILE, &originalLimit);
+
+        // This should not throw "Failed to allocate memory"
+        TestScript prog = "exit 0";
+        assert(execute(prog.path).status == 0);
+    }
+}
+
 @system unittest // Environment variables in spawnProcess().
 {
     // We really should use set /a on Windows, but Wine doesn't support it.