Initial implementation

.gitignore

@@ -1 +1,6 @@
+/dist/
+/*.egg-info/
+/.coverage
+/COLCON_IGNORE
+.*.swp
 __pycache__/

README.rst

@@ -5,3 +5,24 @@ Extensions for `colcon-core <https://github.com/colcon/colcon-core>`_ to work wi
 
 **:warning: This is a prototype package, and isn't ready for widespread use.**
 **Please continue using the Python package support in** ``colcon-core`` **and** ``colcon-python-setup-py`` **until these extensions are ready.**
+
+TODO
+----
+* Graceful and informational error handling
+* More tests
+
+Idiosyncrasies
+--------------
+* For setuptools-based packages, symlink installs do not install data_files.
+* For setuptools-based packages, symlink installs always print warnings to stderr with no good way to suppress them.
+* For poetry-based packages, dependencies expressed in groups (including 'test') are not discovered (use 'test' extra as a workaround).
+
+Using This Prototype
+--------------------
+
+.. code-block::
+
+   $ mkdir -p ~/colcon_pyproject_ws/src && cd ~/colcon_pyproject_ws
+   $ git clone https://github.com/colcon/colcon-python-project.git -b devel src/colcon-python-project
+   $ colcon build
+   $ . install/local_setup.sh

colcon_python_project/argument_parser/python_project.py

@@ -0,0 +1,41 @@
+# Copyright 2023 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+import os
+
+from colcon_core.argument_parser import ArgumentParserDecoratorExtensionPoint
+from colcon_core.plugin_system import satisfies_version
+
+try:
+    from colcon_core.extension_point \
+        import EXTENSION_BLOCKLIST_ENVIRONMENT_VARIABLE as BLOCK_VAR
+except ImportError:
+    from colcon_core.entry_point \
+        import EXTENSION_BLOCKLIST_ENVIRONMENT_VARIABLE as BLOCK_VAR
+
+
+class PythonProjectArgumentParserDecorator(
+    ArgumentParserDecoratorExtensionPoint
+):
+    """Disable extensions which conflict with colcon-python-project."""
+
+    def __init__(self):  # noqa: D107
+        super().__init__()
+        satisfies_version(
+            ArgumentParserDecoratorExtensionPoint.EXTENSION_POINT_VERSION,
+            '^1.0')
+
+    def decorate_argument_parser(self, *, parser):  # noqa: D102
+        blocks = []
+        if os.environ.get(BLOCK_VAR.name, None):
+            blocks = os.environ[BLOCK_VAR.name].split(os.pathsep)
+
+        if 'colcon_core.package_identification.python' not in blocks:
+            blocks.append('colcon_core.package_identification.python')
+
+        if 'colcon_core.package_identification.python_setup_py' not in blocks:
+            blocks.append('colcon_core.package_identification.python_setup_py')
+
+        os.environ[BLOCK_VAR.name] = os.pathsep.join(blocks)
+
+        return parser

colcon_python_project/hook_caller/__init__.py

@@ -0,0 +1,138 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from contextlib import AbstractContextManager
+import os
+import pickle
+import sys
+
+from colcon_core.subprocess import run
+from colcon_python_project.hook_caller import _call_hook
+from colcon_python_project.hook_caller import _list_hooks
+from colcon_python_project.spec import load_and_cache_spec
+
+
+class _SubprocessTransport(AbstractContextManager):
+
+    def __enter__(self):
+        self.child_in, self.parent_out = os.pipe()
+        self.parent_in, self.child_out = os.pipe()
+
+        try:
+            import msvcrt
+        except ImportError:
+            os.set_inheritable(self.child_in, True)
+            self.pass_in = self.child_in
+            os.set_inheritable(self.child_out, True)
+            self.pass_out = self.child_out
+        else:
+            self.pass_in = msvcrt.get_osfhandle(self.child_in)
+            os.set_handle_inheritable(self.pass_in, True)
+            self.pass_out = msvcrt.get_osfhandle(self.child_out)
+            os.set_handle_inheritable(self.pass_out, True)
+
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        os.close(self.parent_out)
+        os.close(self.parent_in)
+        os.close(self.child_out)
+        os.close(self.child_in)
+
+
+class AsyncHookCaller:
+    """Calls PEP 517 style hooks asynchronously in a new process."""
+
+    def __init__(
+        self, backend_name, *, project_path=None, env=None,
+        stdout_callback=None, stderr_callback=None,
+    ):
+        """
+        Initialize a new AsyncHookCaller.
+
+        :param backend_name: The name of the PEP 517 build backend.
+        :param project_path: Path to the project's root directory.
+        :param env: Environment variables to use when invoking hooks.
+        :param stdout_callback: Callback for stdout from the hook invocation.
+        :param stderr_callback: Callback for stderr from the hook invocation.
+        """
+        self._backend_name = backend_name
+        self._project_path = str(project_path) if project_path else None
+        self._env = dict(env if env is not None else os.environ)
+        self._stdout_callback = stdout_callback
+        self._stderr_callback = stderr_callback
+
+    @property
+    def backend_name(self):
+        """Get the name of the backend to call hooks on."""
+        return self._backend_name
+
+    @property
+    def env(self):
+        """Get the environment variables to use when invoking hooks."""
+        return self._env
+
+    async def list_hooks(self):
+        """
+        Call into the backend to list implemented hooks.
+
+        This function lists all callable methods on the backend, which may
+        include more than just the hook names.
+
+        :returns: List of hook names.
+        """
+        args = [
+            sys.executable, _list_hooks.__file__,
+            self._backend_name]
+        process = await run(
+            args, None, self._stderr_callback,
+            cwd=self._project_path, env=self.env,
+            capture_output=True)
+        process.check_returncode()
+        hook_names = [
+            line.strip().decode() for line in process.stdout.splitlines()]
+        return [
+            hook for hook in hook_names if hook and not hook.startswith('_')]
+
+    async def call_hook(self, hook_name, **kwargs):
+        """
+        Call the given hook with given arguments.
+
+        :param hook_name: Name of the hook to call.
+        """
+        with _SubprocessTransport() as transport:
+            args = [
+                sys.executable, _call_hook.__file__,
+                self._backend_name, hook_name,
+                str(transport.pass_in), str(transport.pass_out)]
+            with os.fdopen(os.dup(transport.parent_out), 'wb') as f:
+                pickle.dump(kwargs, f)
+            have_callbacks = self._stdout_callback or self._stderr_callback
+            process = await run(
+                args, self._stdout_callback, self._stderr_callback,
+                cwd=self._project_path, env=self.env, close_fds=False,
+                capture_output=not have_callbacks)
+            process.check_returncode()
+            with os.fdopen(os.dup(transport.parent_in), 'rb') as f:
+                res = pickle.load(f)
+            return res
+
+
+def get_hook_caller(desc, **kwargs):
+    """
+    Create a new AsyncHookCaller instance for a package descriptor.
+
+    :param desc: The package descriptor
+    """
+    spec = load_and_cache_spec(desc)
+    backend_path = spec['build-system'].get('backend-path')
+    if backend_path:
+        # TODO(cottsay): This isn't *technically* the beginning of sys.path
+        #                as PEP 517 calls for, but it's pretty darn close.
+        kwargs['env'] = dict(kwargs.get('env', os.environ))
+        pythonpath = kwargs['env'].get('PYTHONPATH', '')
+        kwargs['env']['PYTHONPATH'] = os.pathsep.join(
+            backend_path + ([pythonpath] if pythonpath else []))
+    return AsyncHookCaller(
+        spec['build-system']['build-backend'],
+        project_path=desc.path, **kwargs)

colcon_python_project/hook_caller/_call_hook.py

@@ -0,0 +1,29 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from importlib import import_module
+import os
+import pickle
+import sys
+
+
+if __name__ == '__main__':
+    backend_name, hook_name, child_in, child_out = sys.argv[1:]
+    try:
+        import msvcrt
+    except ImportError:
+        pass
+    else:
+        child_in = msvcrt.open_osfhandle(int(child_in), os.O_RDONLY)
+        child_out = msvcrt.open_osfhandle(int(child_out), 0)
+    if ':' in backend_name:
+        backend_module_name, backend_object_name = backend_name.split(':', 2)
+        backend_module = import_module(backend_module_name)
+        backend = getattr(backend_module, backend_object_name)
+    else:
+        backend = import_module(backend_name)
+    with os.fdopen(int(child_in), 'rb') as f:
+        kwargs = pickle.load(f) or {}
+    res = getattr(backend, hook_name)(**kwargs)
+    with os.fdopen(int(child_out), 'wb') as f:
+        pickle.dump(res, f)

colcon_python_project/hook_caller/_list_hooks.py

@@ -0,0 +1,19 @@
+# Copyright 2023 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from importlib import import_module
+import sys
+
+
+if __name__ == '__main__':
+    backend_name = sys.argv[1]
+    if ':' in backend_name:
+        backend_module_name, backend_object_name = backend_name.split(':', 2)
+        backend_module = import_module(backend_module_name)
+        backend = getattr(backend_module, backend_object_name)
+    else:
+        backend = import_module(backend_name)
+
+    for attr in dir(backend):
+        if callable(getattr(backend, attr)):
+            print(attr)

colcon_python_project/hook_caller_decorator/__init__.py

@@ -0,0 +1,134 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+import traceback
+
+from colcon_core.logging import colcon_logger
+from colcon_core.plugin_system import instantiate_extensions
+from colcon_core.plugin_system import order_extensions_by_priority
+from colcon_python_project.hook_caller import get_hook_caller
+
+logger = colcon_logger.getChild(__name__)
+
+
+class HookCallerDecoratorExtensionPoint:
+    """
+    The interface for PEP 517 hook caller decorator extensions.
+
+    For each instance the attribute `HOOK_CALLER_DECORATOR_NAME` is being
+    set to the basename of the entry point registering the extension.
+    """
+
+    """The version of the hook caller decorator extension interface."""
+    EXTENSION_POINT_VERSION = '1.0'
+
+    """The default priority of hook caller decorator extensions."""
+    PRIORITY = 100
+
+    def decorate_hook_caller(self, *, hook_caller):
+        """
+        Decorate a hook caller to perform additional functionality.
+
+        This method must be overridden in a subclass.
+
+        :param hook_caller: The hook caller
+        :returns: A decorator
+        """
+        raise NotImplementedError()
+
+
+def get_hook_caller_extensions():
+    """
+    Get the available hook caller decorator extensions.
+
+    The extensions are ordered by their priority and entry point name.
+
+    :rtype: OrderedDict
+    """
+    extensions = instantiate_extensions(__name__)
+    for name, extension in extensions.items():
+        extension.HOOK_CALLER_DECORATOR_NAME = name
+    return order_extensions_by_priority(extensions)
+
+
+def decorate_hook_caller(hook_caller):
+    """
+    Decorate the hook caller using hook caller decorator extensions.
+
+    :param hook_caller: The hook caller
+
+    :returns: The decorated parser
+    """
+    extensions = get_hook_caller_extensions()
+    for extension in extensions.values():
+        logger.log(
+            1, 'decorate_hook_caller() %s',
+            extension.HOOK_CALLER_DECORATOR_NAME)
+        try:
+            decorated_hook_caller = extension.decorate_hook_caller(
+                hook_caller=hook_caller)
+            assert hasattr(decorated_hook_caller, 'call_hook'), \
+                'decorate_hook_caller() should return something to call hooks'
+        except Exception as e:  # noqa: F841
+            # catch exceptions raised in decorator extension
+            exc = traceback.format_exc()
+            logger.error(
+                'Exception in hook caller decorator extension '
+                f"'{extension.HOOK_CALLER_DECORATOR_NAME}': {e}\n{exc}")
+            # skip failing extension, continue with next one
+        else:
+            hook_caller = decorated_hook_caller
+
+    return hook_caller
+
+
+def get_decorated_hook_caller(desc, **kwargs):
+    """
+    Create and decorate a hook caller instance for a package descriptor.
+
+    :param desc: The package descriptor
+    """
+    hook_caller = get_hook_caller(desc, **kwargs)
+    decorated_hook_caller = decorate_hook_caller(hook_caller)
+    return decorated_hook_caller
+
+
+# TODO(cottsay): Promote to colcon_core and use in ArgumentParserDecorator
+class GenericDecorator:
+    """A generic class decorator."""
+
+    def __init__(self, decoree, **kwargs):
+        """
+        Create a new decorated class instance.
+
+        :param decoree: The instance to decorate
+        :param **kwargs: The keyword arguments are set as attributes on this
+          instance
+        """
+        assert '_decoree' not in kwargs
+        kwargs['_decoree'] = decoree
+        for k, v in kwargs.items():
+            self.__dict__[k] = v
+
+    def __getattr__(self, name):
+        """
+        Get an attribute from this decorator if it exists or the decoree.
+
+        :param str name: The name of the attribute
+        :returns: The attribute value
+        """
+        return getattr(self.__dict__['_decoree'], name)
+
+    def __setattr__(self, name, value):
+        """
+        Set an attribute value on this decorator if it exists or the decoree.
+
+        :param str name: The name of the attribute
+        :param value: The attribute value
+        """
+        # overwrite existing attribute
+        if name in self.__dict__:
+            self.__dict__[name] = value
+            return
+        # set attribute on decoree
+        setattr(self.__dict__['_decoree'], name, value)