gh-161456: refresh C stack limits after setrlimit(RLIMIT_STACK)

Author: NekoAsakura

Include/internal/pycore_ceval.h

@@ -245,6 +245,7 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) {
 }
 
 PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate);
+PyAPI_FUNC(void) _Py_UpdateRecursionLimits(PyThreadState *tstate);
 
 // Export for test_peg_generator
 PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(

Lib/test/test_resource.py

@@ -1,9 +1,11 @@
 import contextlib
 import sys
+import textwrap
 import unittest
 from test import support
 from test.support import import_helper
 from test.support import os_helper
+from test.support import script_helper
 
 resource = import_helper.import_module('resource')
 
@@ -234,6 +236,35 @@ def __getitem__(self, key):
         self.assertEqual(resource.prlimit(0, resource.RLIMIT_AS, BadSeq()),
                          limits)
 
+    @unittest.skipUnless(sys.platform == 'linux', 'Linux only')
+    @unittest.skipIf(support.linked_to_musl(),
+                     'no C stack detection on musl, gh-159876')
+    @unittest.skipUnless(hasattr(resource, 'RLIMIT_STACK'),
+                         'requires resource.RLIMIT_STACK')
+    def test_lowered_stack_limit(self):
+        code = textwrap.dedent('''
+            import resource, sys
+            soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
+            limits = (1024 * 1024, hard)
+            if sys.argv[1] == 'setrlimit':
+                resource.setrlimit(resource.RLIMIT_STACK, limits)
+            else:
+                resource.prlimit(0, resource.RLIMIT_STACK, limits)
+            value = None
+            for _ in range(100_000):
+                value = [value]
+            try:
+                repr(value)
+            except RecursionError:
+                pass
+            del value
+        ''')
+        for func in 'setrlimit', 'prlimit':
+            with self.subTest(func=func):
+                if not hasattr(resource, func):
+                    self.skipTest(f'requires resource.{func}')
+                script_helper.assert_python_ok('-c', code, func)
+
 
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Library/2026-11-05-21-28-38.gh-issue-161456.PfMqV7.rst

@@ -0,0 +1,2 @@
+Fix a crash on deep recursion after :func:`resource.setrlimit` or
+:func:`resource.prlimit` lowers :data:`~resource.RLIMIT_STACK`.

Modules/resource.c

@@ -3,6 +3,8 @@
 #endif
 
 #include "Python.h"
+#include "pycore_ceval.h"         // _Py_UpdateRecursionLimits()
+#include "pycore_pystate.h"       // _PyThreadState_GET()
 #include <errno.h>                // errno
 #include <string.h>
 #include <sys/resource.h>         // getrusage()
@@ -283,6 +285,11 @@ resource_setrlimit_impl(PyObject *module, int resource, PyObject *limits)
             PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
     }
+#ifdef RLIMIT_STACK
+    if (resource == RLIMIT_STACK) {
+        _Py_UpdateRecursionLimits(_PyThreadState_GET());
+    }
+#endif
     Py_RETURN_NONE;
 }
 
@@ -335,6 +342,13 @@ resource_prlimit_impl(PyObject *module, pid_t pid, int resource,
         }
         return NULL;
     }
+#ifdef RLIMIT_STACK
+    if (limits != Py_None && resource == RLIMIT_STACK
+        && (pid == 0 || pid == getpid()))
+    {
+        _Py_UpdateRecursionLimits(_PyThreadState_GET());
+    }
+#endif
     return rlimit2py(old_limit);
 }
 #endif /* HAVE_PRLIMIT */

Python/ceval.c

@@ -176,20 +176,35 @@ tstate_set_stack(PyThreadState *tstate,
 }
 
 
-void
-_Py_InitializeRecursionLimits(PyThreadState *tstate)
+static void
+set_recursion_limits(PyThreadState *tstate, uintptr_t min_size)
 {
     uintptr_t base, top;
     uintptr_t here_addr = _Py_get_machine_stack_pointer();
     hardware_stack_limits(&base, &top, here_addr);
     assert(top != 0);
+    if (top - base < min_size) {
+        return;
+    }
 
     tstate_set_stack(tstate, base, top);
     _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate;
     ts->c_stack_init_base = base;
     ts->c_stack_init_top = top;
 }
 
+void
+_Py_InitializeRecursionLimits(PyThreadState *tstate)
+{
+    set_recursion_limits(tstate, 0);
+}
+
+void
+_Py_UpdateRecursionLimits(PyThreadState *tstate)
+{
+    set_recursion_limits(tstate, _PyOS_MIN_STACK_SIZE);
+}
+
 
 int
 PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate,