Fix setting of recursion limit

src/core/IronPython/Runtime/PythonContext.cs

@@ -150,7 +150,7 @@ public sealed partial class PythonContext : LanguageContext {
         internal readonly object _codeCleanupLock = new object();
         internal readonly object _codeUpdateLock = new object();
         internal int _codeCount, _nextCodeCleanup = 200;
-        private int _recursionLimit;
+        private int _recursionLimit = int.MaxValue; // not enforced until set from the options in the constructor
 
         private readonly object _indexBindersLock = new object(); // https://github.com/IronLanguages/ironpython3/issues/1596
 
@@ -348,9 +348,10 @@ public int RecursionLimit {
                 }
 
                 lock (_codeUpdateLock) {
+                    bool wasEnforced = _recursionLimit != int.MaxValue;
                     _recursionLimit = value;
 
-                    if ((_recursionLimit == int.MaxValue) != (value == int.MaxValue)) {
+                    if (wasEnforced != (value != int.MaxValue)) {
                         // recursion setting has changed, we need to update all of our
                         // function codes to enforce or un-enforce recursion.
                         FunctionCode.UpdateAllCode(this);

tests/suite/modules/system_related/test_sys.py

@@ -361,4 +361,22 @@ def test_builtin_module_names(self):
             See issue https://github.com/IronLanguages/ironpython3/issues/875 for full details '''
         self.assertTrue(sys.builtin_module_names == tuple(sorted(sys.builtin_module_names)))
 
+    def test_setrecursionlimit_existing_function(self):
+        # Setting a recursion limit is enforced also for functions that were already called before
+        # (in a new process, where IronPython enforces no limit by default)
+        import subprocess
+        code = """import sys
+def f(n):
+    return f(n - 1) if n else 0
+f(10)
+sys.setrecursionlimit(50)
+try:
+    f(100)
+    print(False)
+except RecursionError:
+    print(True)
+"""
+        out = subprocess.check_output([sys.executable, '-c', code])
+        self.assertEqual(out.strip(), b'True')
+
 run_test(__name__)