gh-150436: Skip subprocess test on STATUS_DLL_INIT_FAILED (#150704)

If a subprocess spawned with CREATE_NEW_CONSOLE creation flag fails
with STATUS_DLL_INIT_FAILED return code, skip the test. It's likely a
memory allocation failure in the desktop heap memory which caused the
DLL init failure.

Author: vstinner

Lib/test/support/__init__.py

@@ -3318,3 +3318,18 @@ def control_characters_c0() -> list[str]:
 _ROOT_IN_POSIX = hasattr(os, 'geteuid') and os.geteuid() == 0
 requires_root_user = unittest.skipUnless(_ROOT_IN_POSIX, "test needs root privilege")
 requires_non_root_user = unittest.skipIf(_ROOT_IN_POSIX, "test needs non-root account")
+
+
+STATUS_DLL_INIT_FAILED = 0xC0000142
+def skip_on_low_desktop_heap_memory_subprocess(returncode):
+    if sys.platform not in ('win32', 'cygwin'):
+        return
+    # On Windows, STATUS_DLL_INIT_FAILED is a generic error code that could
+    # come from any of the DLLs being loaded when a new Python process is
+    # created. In practice, it's likely a memory allocation failure in the
+    # desktop heap memory which caused the DLL init failure, especially on
+    # process created with CREATE_NEW_CONSOLE creation flag. See the article:
+    # https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/desktop-heap-limitation-out-of-memory
+    if returncode == STATUS_DLL_INIT_FAILED:
+        raise unittest.SkipTest('gh-150436: DLL init failed, likely because '
+                                'of low desktop heap memory')

Lib/test/test_cmd_line.py

@@ -1036,6 +1036,7 @@ def test_python_legacy_windows_stdio(self):
         p = subprocess.run([sys.executable, "-c", code],
                            creationflags=subprocess.CREATE_NEW_CONSOLE,
                            env=env)
+        support.skip_on_low_desktop_heap_memory_subprocess(p.returncode)
         self.assertEqual(p.returncode, 0)
 
         # Then test that FIleIO is used when PYTHONLEGACYWINDOWSSTDIO is set.
@@ -1044,6 +1045,7 @@ def test_python_legacy_windows_stdio(self):
         p = subprocess.run([sys.executable, "-c", code],
                            creationflags=subprocess.CREATE_NEW_CONSOLE,
                            env=env)
+        support.skip_on_low_desktop_heap_memory_subprocess(p.returncode)
         self.assertEqual(p.returncode, 0)
 
     @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()),

Lib/test/test_msvcrt.py

@@ -4,6 +4,7 @@
 import unittest
 from textwrap import dedent
 
+from test import support
 from test.support import os_helper, requires_resource
 from test.support.os_helper import TESTFN, TESTFN_ASCII
 
@@ -67,8 +68,12 @@ def run_in_separated_process(self, code):
         # Run test in a separated process to avoid stdin conflicts.
         # See: gh-110147
         cmd = [sys.executable, '-c', code]
-        subprocess.run(cmd, check=True, capture_output=True,
-                       creationflags=subprocess.CREATE_NEW_CONSOLE)
+        try:
+            subprocess.run(cmd, check=True, capture_output=True,
+                           creationflags=subprocess.CREATE_NEW_CONSOLE)
+        except subprocess.CalledProcessError as exc:
+            support.skip_on_low_desktop_heap_memory_subprocess(exc.returncode)
+            raise
 
     def test_kbhit(self):
         code = dedent('''

Lib/test/test_subprocess.py

@@ -3765,13 +3765,17 @@ def test_startupinfo_copy(self):
             self.assertEqual(startupinfo.wShowWindow, subprocess.SW_HIDE)
             self.assertEqual(startupinfo.lpAttributeList, {"handle_list": []})
 
+    # CREATE_NEW_CONSOLE creates a "popup" window.
+    @support.requires_resource('gui')
     def test_creationflags(self):
         # creationflags argument
         CREATE_NEW_CONSOLE = 16
         sys.stderr.write("    a DOS box should flash briefly ...\n")
-        subprocess.call(sys.executable +
-                        ' -c "import time; time.sleep(0.25)"',
-                        creationflags=CREATE_NEW_CONSOLE)
+        rc = subprocess.call(sys.executable +
+                             ' -c "import time; time.sleep(0.25)"',
+                             creationflags=CREATE_NEW_CONSOLE)
+        support.skip_on_low_desktop_heap_memory_subprocess(rc)
+        self.assertEqual(rc, 0)
 
     def test_invalid_args(self):
         # invalid arguments should raise ValueError