bpo-39763: distutils.spawn now uses subprocess (GH-18743)

Reimplement distutils.spawn.spawn() function with the subprocess
module.

setup.py now uses a basic implementation of the subprocess module if
the subprocess module is not available: before required C extension
modules are built.

Author: vstinner

Lib/distutils/spawn.py

@@ -8,11 +8,18 @@
 
 import sys
 import os
+import subprocess
 
 from distutils.errors import DistutilsPlatformError, DistutilsExecError
 from distutils.debug import DEBUG
 from distutils import log
 
+
+if sys.platform == 'darwin':
+    _cfg_target = None
+    _cfg_target_split = None
+
+
 def spawn(cmd, search_path=1, verbose=0, dry_run=0):
     """Run another program, specified as a command list 'cmd', in a new process.
 
@@ -32,64 +39,16 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0):
     # cmd is documented as a list, but just in case some code passes a tuple
     # in, protect our %-formatting code against horrible death
     cmd = list(cmd)
-    if os.name == 'posix':
-        _spawn_posix(cmd, search_path, dry_run=dry_run)
-    elif os.name == 'nt':
-        _spawn_nt(cmd, search_path, dry_run=dry_run)
-    else:
-        raise DistutilsPlatformError(
-              "don't know how to spawn programs on platform '%s'" % os.name)
-
-def _nt_quote_args(args):
-    """Quote command-line arguments for DOS/Windows conventions.
-
-    Just wraps every argument which contains blanks in double quotes, and
-    returns a new argument list.
-    """
-    # XXX this doesn't seem very robust to me -- but if the Windows guys
-    # say it'll work, I guess I'll have to accept it.  (What if an arg
-    # contains quotes?  What other magic characters, other than spaces,
-    # have to be escaped?  Is there an escaping mechanism other than
-    # quoting?)
-    for i, arg in enumerate(args):
-        if ' ' in arg:
-            args[i] = '"%s"' % arg
-    return args
-
-def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0):
-    executable = cmd[0]
-    cmd = _nt_quote_args(cmd)
-    if search_path:
-        # either we find one or it stays the same
-        executable = find_executable(executable) or executable
-    log.info(' '.join([executable] + cmd[1:]))
-    if not dry_run:
-        # spawn for NT requires a full path to the .exe
-        try:
-            rc = os.spawnv(os.P_WAIT, executable, cmd)
-        except OSError as exc:
-            # this seems to happen when the command isn't found
-            if not DEBUG:
-                cmd = executable
-            raise DistutilsExecError(
-                  "command %r failed: %s" % (cmd, exc.args[-1]))
-        if rc != 0:
-            # and this reflects the command running but failing
-            if not DEBUG:
-                cmd = executable
-            raise DistutilsExecError(
-                  "command %r failed with exit status %d" % (cmd, rc))
-
-if sys.platform == 'darwin':
-    _cfg_target = None
-    _cfg_target_split = None
 
-def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
     log.info(' '.join(cmd))
     if dry_run:
         return
-    executable = cmd[0]
-    exec_fn = search_path and os.execvp or os.execv
+
+    if search_path:
+        executable = find_executable(cmd[0])
+        if executable is not None:
+            cmd[0] = executable
+
     env = None
     if sys.platform == 'darwin':
         global _cfg_target, _cfg_target_split
@@ -111,60 +70,17 @@ def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
                 raise DistutilsPlatformError(my_msg)
             env = dict(os.environ,
                        MACOSX_DEPLOYMENT_TARGET=cur_target)
-            exec_fn = search_path and os.execvpe or os.execve
-    pid = os.fork()
-    if pid == 0: # in the child
-        try:
-            if env is None:
-                exec_fn(executable, cmd)
-            else:
-                exec_fn(executable, cmd, env)
-        except OSError as e:
-            if not DEBUG:
-                cmd = executable
-            sys.stderr.write("unable to execute %r: %s\n"
-                             % (cmd, e.strerror))
-            os._exit(1)
 
+    proc = subprocess.Popen(cmd, env=env)
+    proc.wait()
+    exitcode = proc.returncode
+
+    if exitcode:
         if not DEBUG:
-            cmd = executable
-        sys.stderr.write("unable to execute %r for unknown reasons" % cmd)
-        os._exit(1)
-    else: # in the parent
-        # Loop until the child either exits or is terminated by a signal
-        # (ie. keep waiting if it's merely stopped)
-        while True:
-            try:
-                pid, status = os.waitpid(pid, 0)
-            except OSError as exc:
-                if not DEBUG:
-                    cmd = executable
-                raise DistutilsExecError(
-                      "command %r failed: %s" % (cmd, exc.args[-1]))
-            if os.WIFSIGNALED(status):
-                if not DEBUG:
-                    cmd = executable
-                raise DistutilsExecError(
-                      "command %r terminated by signal %d"
-                      % (cmd, os.WTERMSIG(status)))
-            elif os.WIFEXITED(status):
-                exit_status = os.WEXITSTATUS(status)
-                if exit_status == 0:
-                    return   # hey, it succeeded!
-                else:
-                    if not DEBUG:
-                        cmd = executable
-                    raise DistutilsExecError(
-                          "command %r failed with exit status %d"
-                          % (cmd, exit_status))
-            elif os.WIFSTOPPED(status):
-                continue
-            else:
-                if not DEBUG:
-                    cmd = executable
-                raise DistutilsExecError(
-                      "unknown error executing %r: termination status %d"
-                      % (cmd, status))
+            cmd = cmd[0]
+        raise DistutilsExecError(
+              "command %r failed with exit code %s" % (cmd, exitcode))
+
 
 def find_executable(executable, path=None):
     """Tries to find 'executable' in the directories listed in 'path'.

Lib/distutils/tests/test_spawn.py

@@ -8,7 +8,6 @@
 from test import support as test_support
 
 from distutils.spawn import find_executable
-from distutils.spawn import _nt_quote_args
 from distutils.spawn import spawn
 from distutils.errors import DistutilsExecError
 from distutils.tests import support
@@ -17,16 +16,6 @@ class SpawnTestCase(support.TempdirManager,
                     support.LoggingSilencer,
                     unittest.TestCase):
 
-    def test_nt_quote_args(self):
-
-        for (args, wanted) in ((['with space', 'nospace'],
-                                ['"with space"', 'nospace']),
-                               (['nochange', 'nospace'],
-                                ['nochange', 'nospace'])):
-            res = _nt_quote_args(args)
-            self.assertEqual(res, wanted)
-
-
     @unittest.skipUnless(os.name in ('nt', 'posix'),
                          'Runs only under posix or nt')
     def test_spawn(self):

Misc/NEWS.d/next/Build/2020-03-02-14-44-09.bpo-39763.GGEwhH.rst

@@ -0,0 +1,3 @@
+setup.py now uses a basic implementation of the :mod:`subprocess` module if
+the :mod:`subprocess` module is not available: before required C extension
+modules are built.

Misc/NEWS.d/next/Library/2020-03-02-14-43-19.bpo-39763.5a822c.rst

@@ -0,0 +1,2 @@
+Reimplement :func:`distutils.spawn.spawn` function with the
+:mod:`subprocess` module.

setup.py

@@ -10,6 +10,61 @@
 import sysconfig
 from glob import glob
 
+
+try:
+    import subprocess
+    del subprocess
+    SUBPROCESS_BOOTSTRAP = False
+except ImportError:
+    SUBPROCESS_BOOTSTRAP = True
+
+    # Bootstrap Python: distutils.spawn uses subprocess to build C extensions,
+    # subprocess requires C extensions built by setup.py like _posixsubprocess.
+    #
+    # Basic subprocess implementation for POSIX (setup.py is not used on
+    # Windows) which only uses os functions. Only implement features required
+    # by distutils.spawn.
+    #
+    # It is dropped from sys.modules as soon as all C extension modules
+    # are built.
+    class Popen:
+        def __init__(self, cmd, env=None):
+            self._cmd = cmd
+            self._env = env
+            self.returncode = None
+
+        def wait(self):
+            pid = os.fork()
+            if pid == 0:
+                # Child process
+                try:
+                    if self._env is not None:
+                        os.execve(self._cmd[0], self._cmd, self._env)
+                    else:
+                        os.execv(self._cmd[0], self._cmd)
+                finally:
+                    os._exit(1)
+            else:
+                # Parent process
+                pid, status = os.waitpid(pid, 0)
+                if os.WIFSIGNALED(status):
+                    self.returncode = -os.WTERMSIG(status)
+                elif os.WIFEXITED(status):
+                    self.returncode = os.WEXITSTATUS(status)
+                elif os.WIFSTOPPED(status):
+                    self.returncode = -os.WSTOPSIG(sts)
+                else:
+                    # Should never happen
+                    raise Exception("Unknown child exit status!")
+
+            return self.returncode
+
+    mod = type(sys)('subprocess')
+    mod.Popen = Popen
+    sys.modules['subprocess'] = mod
+    del mod
+
+
 from distutils import log
 from distutils.command.build_ext import build_ext
 from distutils.command.build_scripts import build_scripts
@@ -391,6 +446,11 @@ def build_extensions(self):
 
         build_ext.build_extensions(self)
 
+        if SUBPROCESS_BOOTSTRAP:
+            # Drop our custom subprocess module:
+            # use the newly built subprocess module
+            del sys.modules['subprocess']
+
         for ext in self.extensions:
             self.check_extension_import(ext)