From f6ee0c898f4ae7ec498e5835ab7cb05fd45a6675 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 11:58:11 -0800 Subject: [PATCH 01/67] Add mechanism to read pyproject.toml specs --- colcon_python_project/spec.py | 48 +++++++++++++++++++++++++++++++++++ setup.cfg | 1 + stdeb.cfg | 2 +- test/spell_check.words | 5 ++++ 4 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 colcon_python_project/spec.py diff --git a/colcon_python_project/spec.py b/colcon_python_project/spec.py new file mode 100644 index 0000000..ab5eb6f --- /dev/null +++ b/colcon_python_project/spec.py @@ -0,0 +1,48 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +try: + from tomllib import load as toml_load +except ImportError: + from tomli import load as toml_load + + +SPEC_NAME = 'pyproject.toml' + +_DEFAULT_BUILD_SYSTEM = { + 'build-backend': 'setuptools.build_meta:__legacy__', + 'requires': ['setuptools >= 40.8.0', 'wheel'], +} + + +def load_spec(project_path): + """ + Load build system specifications for a Python project. + + :param project_path: Path to the root directory of the project + """ + spec_file = project_path / SPEC_NAME + try: + with spec_file.open('rb') as f: + spec = toml_load(f) + except FileNotFoundError: + spec = {} + + spec.setdefault('build-system', _DEFAULT_BUILD_SYSTEM) + + return spec + + +def load_and_cache_spec(desc): + """ + Get the cached spec for a package descriptor. + + If the spec has not been loaded yet, load and cache it. + + :param desc: The package descriptor + """ + spec = desc.metadata.get('python_project_spec') + if spec is None: + spec = load_spec(desc.path) + desc.metadata['python_project_spec'] = spec + return spec diff --git a/setup.cfg b/setup.cfg index 432bfd1..0c41654 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ keywords = colcon [options] install_requires = colcon-core + tomli>=1.0.0;python_version < "3.11" packages = find: zip_safe = true diff --git a/stdeb.cfg b/stdeb.cfg index ba1b742..052cfe9 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,5 +1,5 @@ [colcon-python-project] No-Python2: -Depends3: python3-colcon-core +Depends3: python3-colcon-core, python3-tomli (>= 1.0.0) Suite: jammy bullseye X-Python3-Version: >= 3.6 diff --git a/test/spell_check.words b/test/spell_check.words index 91e2768..1982d28 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -1,8 +1,13 @@ apache +backend colcon iterdir pathlib +pyproject pytest scspell setuptools thomas +toml +tomli +tomllib From 5170d47bb508719bda36620e3f3bb1095a2f6032 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 12:17:47 -0800 Subject: [PATCH 02/67] Implement PEP 621 identification and augmentation --- .../package_augmentation/__init__.py | 0 .../package_augmentation/pep621.py | 67 +++++++++++++++++++ .../package_identification/__init__.py | 0 .../package_identification/pep621.py | 44 ++++++++++++ setup.cfg | 4 ++ test/spell_check.words | 4 ++ 6 files changed, 119 insertions(+) create mode 100644 colcon_python_project/package_augmentation/__init__.py create mode 100644 colcon_python_project/package_augmentation/pep621.py create mode 100644 colcon_python_project/package_identification/__init__.py create mode 100644 colcon_python_project/package_identification/pep621.py diff --git a/colcon_python_project/package_augmentation/__init__.py b/colcon_python_project/package_augmentation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/package_augmentation/pep621.py b/colcon_python_project/package_augmentation/pep621.py new file mode 100644 index 0000000..eb53777 --- /dev/null +++ b/colcon_python_project/package_augmentation/pep621.py @@ -0,0 +1,67 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import logging + +from colcon_core.package_augmentation \ + import PackageAugmentationExtensionPoint +from colcon_core.package_augmentation.python import \ + create_dependency_descriptor +from colcon_core.plugin_system import satisfies_version +from colcon_python_project.spec import load_and_cache_spec + + +class PEP621PackageAugmentation(PackageAugmentationExtensionPoint): + """Augment Python packages with `pyproject.toml` using a build backend.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + # avoid debug message from asyncio when colcon uses debug log level + asyncio_logger = logging.getLogger('asyncio') + asyncio_logger.setLevel(logging.INFO) + + def augment_package( # noqa: D102 + self, desc, *, additional_argument_names=None + ): + if desc.type != 'python.project': + return + + spec = load_and_cache_spec(desc) + desc.dependencies.setdefault('build', set()) + desc.dependencies['build'].update( + create_dependency_descriptor(d) + for d in spec['build-system'].get('requires') or ()) + + project = spec.get('project', {}) + desc.dependencies.setdefault('run', set()) + desc.dependencies['run'].update( + create_dependency_descriptor(d) + for d in project.get('dependencies') or ()) + + optional_deps = project.get('optional-dependencies', {}) + desc.dependencies.setdefault('test', set()) + desc.dependencies['test'].update( + create_dependency_descriptor(d) + for d in optional_deps.get('test') or ()) + + version = project.get('version') + if version: + desc.metadata['version'] = version + + maintainers = project.get('maintainers') + if not maintainers: + maintainers = project.get('authors') + if maintainers: + desc.metadata.setdefault('maintainers', []) + for entry in maintainers: + email = entry.get('email') + if not email: + continue + name = entry.get('name') + rfc822 = f'{name} <{email}>' if name else email + if rfc822 not in desc.metadata['maintainers']: + desc.metadata['maintainers'].append(rfc822) diff --git a/colcon_python_project/package_identification/__init__.py b/colcon_python_project/package_identification/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/package_identification/pep621.py b/colcon_python_project/package_identification/pep621.py new file mode 100644 index 0000000..7c0aa3c --- /dev/null +++ b/colcon_python_project/package_identification/pep621.py @@ -0,0 +1,44 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_core.package_identification import logger +from colcon_core.package_identification \ + import PackageIdentificationExtensionPoint +from colcon_core.plugin_system import satisfies_version +from colcon_python_project.spec import load_and_cache_spec +from colcon_python_project.spec import SPEC_NAME + + +class PEP621PackageIdentification(PackageIdentificationExtensionPoint): + """Identify Python packages with `pyproject.toml` metadata.""" + + # the priority should be higher than the extensions which enhance Python + # project package identification, and should also be hither than those + # using setup.py and/or setup.cfg files + PRIORITY = 150 + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageIdentificationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + def identify(self, desc): # noqa: D102 + if desc.type is not None and desc.type != 'python.project': + return + + spec_file = desc.path / SPEC_NAME + if not spec_file.is_file(): + return + + spec = load_and_cache_spec(desc) + name = spec.get('project', {}).get('name') + if not name: + return + + if desc.name is not None and desc.name != name: + msg = 'Package name already set to different value' + logger.error(msg) + raise RuntimeError(msg) + desc.name = name + desc.type = 'python.project' diff --git a/setup.cfg b/setup.cfg index 0c41654..5d7e68a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,6 +57,10 @@ filterwarnings = junit_suite_name = colcon-python-project [options.entry_points] +colcon_core.package_augmentation = + pep621 = colcon_python_project.package_augmentation.pep621:PEP621PackageAugmentation +colcon_core.package_identification = + pep621 = colcon_python_project.package_identification.pep621:PEP621PackageIdentification [flake8] import-order-style = google diff --git a/test/spell_check.words b/test/spell_check.words index 1982d28..5bc0ad5 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -1,8 +1,12 @@ apache +asyncio backend colcon +deps iterdir +noqa pathlib +plugin pyproject pytest scspell From f863eaee500498bf21596cba68e551614dd2349b Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 12:01:30 -0800 Subject: [PATCH 03/67] Implement PEP 517 hook caller --- colcon_python_project/_call_hook.py | 22 ++++++ colcon_python_project/hook_caller.py | 107 ++++++++++++++++++++++++++ setup.cfg | 1 + test/spell_check.words | 9 +++ test/test_hook_caller_setuptools.py | 109 +++++++++++++++++++++++++++ 5 files changed, 248 insertions(+) create mode 100644 colcon_python_project/_call_hook.py create mode 100644 colcon_python_project/hook_caller.py create mode 100644 test/test_hook_caller_setuptools.py diff --git a/colcon_python_project/_call_hook.py b/colcon_python_project/_call_hook.py new file mode 100644 index 0000000..0bfc8de --- /dev/null +++ b/colcon_python_project/_call_hook.py @@ -0,0 +1,22 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from importlib import import_module +import os +import pickle +import sys + + +if __name__ == '__main__': + backend_name, hook_name, child_in, child_out = sys.argv[1:] + if ':' in backend_name: + backend_module_name, backend_object_name = backend_name.split(':', 2) + backend_module = import_module(backend_module_name) + backend = getattr(backend_module, backend_object_name) + else: + backend = import_module(backend_name) + with os.fdopen(int(child_in), 'rb') as f: + kwargs = pickle.load(f) or {} + res = getattr(backend, hook_name)(**kwargs) + with os.fdopen(int(child_out), 'wb') as f: + pickle.dump(res, f) diff --git a/colcon_python_project/hook_caller.py b/colcon_python_project/hook_caller.py new file mode 100644 index 0000000..6237264 --- /dev/null +++ b/colcon_python_project/hook_caller.py @@ -0,0 +1,107 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from contextlib import AbstractContextManager +from functools import partialmethod +import os +import pickle +import sys + +from colcon_core.subprocess import run +from colcon_python_project.spec import load_and_cache_spec + + +class _SubprocessTransport(AbstractContextManager): + + def __enter__(self): + self.child_in, self.parent_out = os.pipe() + os.set_inheritable(self.child_in, True) + self.parent_in, self.child_out = os.pipe() + os.set_inheritable(self.child_out, True) + return self + + def __exit__(self, exc_type, exc_value, traceback): + os.close(self.parent_out) + os.close(self.parent_in) + os.close(self.child_out) + os.close(self.child_in) + + +class AsyncHookCaller: + """Calls PEP 517 style hooks asynchronously in a new process.""" + + def __init__( + self, backend_name, *, project_path=None, env=None, + stdout_callback=None, stderr_callback=None, + ): + """ + Initialize a new AsyncHookCaller. + + :param backend_name: The name of the PEP 517 build backend. + :param project_path: Path to the project's root directory. + :param env: Environment variables to use when invoking hooks. + :param stdout_callback: Callback for stdout from the hook invocation. + :param stderr_callback: Callback for stderr from the hook invocation. + """ + self._backend_name = backend_name + self._project_path = str(project_path) if project_path else None + self._env = dict(env) if env else None + self._stdout_callback = stdout_callback + self._stderr_callback = stderr_callback + + @property + def backend_name(self): + """Get the name of the backend to call hooks on.""" + return self._backend_name + + async def call_hook(self, hook_name, **kwargs): + """ + Call the given hook with given arguments. + + :param hook_name: Name of the hook to call. + """ + with _SubprocessTransport() as transport: + args = [ + sys.executable, '-m', 'colcon_python_project._call_hook', + self._backend_name, hook_name, + str(transport.child_in), str(transport.child_out)] + with os.fdopen(os.dup(transport.parent_out), 'wb') as f: + pickle.dump(kwargs, f) + have_callbacks = self._stdout_callback or self._stderr_callback + process = await run( + args, self._stdout_callback, self._stderr_callback, + cwd=self._project_path, env=self._env, close_fds=False, + capture_output=not have_callbacks) + process.check_returncode() + with os.fdopen(os.dup(transport.parent_in), 'rb') as f: + res = pickle.load(f) + return res + + # PEP 517 + build_wheel = partialmethod(call_hook, 'build_wheel') + build_sdist = partialmethod(call_hook, 'build_sdist') + get_requires_for_build_wheel = partialmethod( + call_hook, 'get_requires_for_build_wheel') + prepare_metadata_for_build_wheel = partialmethod( + call_hook, 'prepare_metadata_for_build_wheel') + get_requires_for_build_sdist = partialmethod( + call_hook, 'get_requires_for_build_sdist') + + # PEP 660 + build_editable = partialmethod(call_hook, 'build_editable') + get_requires_for_build_editable = partialmethod( + call_hook, 'get_requires_for_build_editable') + prepare_metadata_for_build_editable = partialmethod( + call_hook, 'prepare_metadata_for_build_editable') + + +def get_hook_caller(desc, **kwargs): + """ + Create a new AsyncHookCaller instance for a package descriptor. + + :param desc: The package descriptor + """ + spec = load_and_cache_spec(desc) + return AsyncHookCaller( + spec['build-system']['build-backend'], + project_path=desc.path, **kwargs) diff --git a/setup.cfg b/setup.cfg index 5d7e68a..d580f86 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,6 +46,7 @@ test = pytest pytest-cov scspell3k>=2.2 + setuptools>=40.8.0 [tool:pytest] filterwarnings = diff --git a/test/spell_check.words b/test/spell_check.words index 5bc0ad5..d97d4f5 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -2,16 +2,25 @@ apache asyncio backend colcon +contextlib deps +distinfo +fdopen +functools +importlib iterdir noqa +partialmethod pathlib plugin pyproject pytest +returncode scspell +sdist setuptools thomas toml tomli tomllib +traceback diff --git a/test/test_hook_caller_setuptools.py b/test/test_hook_caller_setuptools.py new file mode 100644 index 0000000..dbb2fa9 --- /dev/null +++ b/test/test_hook_caller_setuptools.py @@ -0,0 +1,109 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_python_project.hook_caller import AsyncHookCaller +import pytest + +_BACKEND_NAME = 'setuptools.build_meta:__legacy__' + + +@pytest.fixture +def mock_project(tmp_path): + with (tmp_path / 'test_project.py').open('w'): + pass + with (tmp_path / 'README').open('w'): + pass + with (tmp_path / 'setup.cfg').open('w') as f: + f.write('\n'.join([ + '[metadata]', + 'name = test-project', + '[options]', + 'packages = find:', + ])) + with (tmp_path / 'setup.py').open('w') as f: + f.write('\n'.join([ + 'from setuptools import setup', + 'setup()', + ])) + with (tmp_path / 'pyproject.toml').open('w') as f: + f.write('\n'.join([ + '[build-system]', + 'requires = ["setuptools>=40.8.0"]', + f'build-backend = "{_BACKEND_NAME}"', + ])) + yield tmp_path + + +@pytest.mark.asyncio +async def test_build_wheel(mock_project, tmp_path): + hook_caller = AsyncHookCaller( + _BACKEND_NAME, project_path=mock_project) + wheel = await hook_caller.build_wheel(wheel_directory=str(tmp_path)) + assert isinstance(wheel, str) + assert (tmp_path / wheel).is_file() + + +@pytest.mark.asyncio +async def test_build_sdist(mock_project, tmp_path): + hook_caller = AsyncHookCaller( + _BACKEND_NAME, project_path=mock_project) + sdist = await hook_caller.build_sdist(sdist_directory=str(tmp_path)) + assert isinstance(sdist, str) + assert (tmp_path / sdist).is_file() + + +@pytest.mark.asyncio +async def test_get_requires_for_build_wheel(mock_project): + hook_caller = AsyncHookCaller( + _BACKEND_NAME, project_path=mock_project) + requires = await hook_caller.get_requires_for_build_wheel() + assert isinstance(requires, list) + + +@pytest.mark.asyncio +async def test_prepare_metadata_for_build_wheel(mock_project, tmp_path): + hook_caller = AsyncHookCaller( + _BACKEND_NAME, project_path=mock_project) + distinfo = await hook_caller.prepare_metadata_for_build_wheel( + metadata_directory=str(tmp_path)) + assert isinstance(distinfo, str) + assert (tmp_path / distinfo).is_dir() + + +@pytest.mark.asyncio +async def test_get_requires_for_build_sdist(mock_project): + hook_caller = AsyncHookCaller( + _BACKEND_NAME, project_path=mock_project) + requires = await hook_caller.get_requires_for_build_sdist() + assert isinstance(requires, list) + + +@pytest.mark.asyncio +@pytest.mark.skip(reason='Insufficient setuptools version') +async def test_build_editable(mock_project, tmp_path): + hook_caller = AsyncHookCaller( + _BACKEND_NAME, project_path=mock_project) + wheel = await hook_caller.build_editable( + wheel_directory=str(tmp_path)) + assert isinstance(wheel, str) + assert (tmp_path / wheel).is_file() + + +@pytest.mark.asyncio +@pytest.mark.skip(reason='Insufficient setuptools version') +async def test_get_requires_for_build_editable(mock_project): + hook_caller = AsyncHookCaller( + _BACKEND_NAME, project_path=mock_project) + requires = await hook_caller.get_requires_for_build_editable() + assert isinstance(requires, list) + + +@pytest.mark.asyncio +@pytest.mark.skip(reason='Insufficient setuptools version') +async def test_prepare_metadata_for_build_editable(mock_project, tmp_path): + hook_caller = AsyncHookCaller( + _BACKEND_NAME, project_path=mock_project) + dist_info = await hook_caller.prepare_metadata_for_build_editable( + metadata_directory=str(tmp_path)) + assert isinstance(dist_info, str) + assert (tmp_path / dist_info).is_dir() From ab45092b5807b099b82842b3c468e5aa0ffd359a Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 12:22:36 -0800 Subject: [PATCH 04/67] Implement decoration extension for enhancing hook calls --- .../hook_caller_decorator/__init__.py | 134 ++++++++++++++++++ .../hook_caller_decorator/setuptools.py | 92 ++++++++++++ setup.cfg | 4 + test/spell_check.words | 5 + 4 files changed, 235 insertions(+) create mode 100644 colcon_python_project/hook_caller_decorator/__init__.py create mode 100644 colcon_python_project/hook_caller_decorator/setuptools.py diff --git a/colcon_python_project/hook_caller_decorator/__init__.py b/colcon_python_project/hook_caller_decorator/__init__.py new file mode 100644 index 0000000..a3c7a1d --- /dev/null +++ b/colcon_python_project/hook_caller_decorator/__init__.py @@ -0,0 +1,134 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import traceback + +from colcon_core.logging import colcon_logger +from colcon_core.plugin_system import instantiate_extensions +from colcon_core.plugin_system import order_extensions_by_priority +from colcon_python_project.hook_caller import get_hook_caller + +logger = colcon_logger.getChild(__name__) + + +class HookCallerDecoratorExtensionPoint: + """ + The interface for PEP 517 hook caller decorator extensions. + + For each instance the attribute `HOOK_CALLER_DECORATOR_NAME` is being + set to the basename of the entry point registering the extension. + """ + + """The version of the hook caller decorator extension interface.""" + EXTENSION_POINT_VERSION = '1.0' + + """The default priority of hook caller decorator extensions.""" + PRIORITY = 100 + + def decorate_hook_caller(self, *, hook_caller): + """ + Decorate a hook caller to perform additional functionality. + + This method must be overridden in a subclass. + + :param hook_caller: The hook caller + :returns: A decorator + """ + raise NotImplementedError() + + +def get_hook_caller_extensions(): + """ + Get the available hook caller decorator extensions. + + The extensions are ordered by their priority and entry point name. + + :rtype: OrderedDict + """ + extensions = instantiate_extensions(__name__) + for name, extension in extensions.items(): + extension.HOOK_CALLER_DECORATOR_NAME = name + return order_extensions_by_priority(extensions) + + +def decorate_hook_caller(hook_caller): + """ + Decorate the hook caller using hook caller decorator extensions. + + :param hook_caller: The hook caller + + :returns: The decorated parser + """ + extensions = get_hook_caller_extensions() + for extension in extensions.values(): + logger.log( + 1, 'decorate_hook_caller() %s', + extension.HOOK_CALLER_DECORATOR_NAME) + try: + decorated_hook_caller = extension.decorate_hook_caller( + hook_caller=hook_caller) + assert hasattr(decorated_hook_caller, 'call_hook'), \ + 'decorate_hook_caller() should return something to call hooks' + except Exception as e: # noqa: F841 + # catch exceptions raised in decorator extension + exc = traceback.format_exc() + logger.error( + 'Exception in hook caller decorator extension ' + f"'{extension.HOOK_CALLER_DECORATOR_NAME}': {e}\n{exc}") + # skip failing extension, continue with next one + else: + hook_caller = decorated_hook_caller + + return hook_caller + + +def get_decorated_hook_caller(desc, **kwargs): + """ + Create and decorate a hook caller instance for a package descriptor. + + :param desc: The package descriptor + """ + hook_caller = get_hook_caller(desc, **kwargs) + decorated_hook_caller = decorate_hook_caller(hook_caller) + return decorated_hook_caller + + +# TODO(cottsay): Promote to colcon_core and use in ArgumentParserDecorator +class GenericDecorator: + """A generic class decorator.""" + + def __init__(self, decoree, **kwargs): + """ + Create a new decorated class instance. + + :param decoree: The instance to decorate + :param **kwargs: The keyword arguments are set as attributes on this + instance + """ + assert '_decoree' not in kwargs + kwargs['_decoree'] = decoree + for k, v in kwargs.items(): + self.__dict__[k] = v + + def __getattr__(self, name): + """ + Get an attribute from this decorator if it exists or the decoree. + + :param str name: The name of the attribute + :returns: The attribute value + """ + return getattr(self.__dict__['_decoree'], name) + + def __setattr__(self, name, value): + """ + Set an attribute value on this decorator if it exists or the decoree. + + :param str name: The name of the attribute + :param value: The attribute value + """ + # overwrite existing attribute + if name in self.__dict__: + self.__dict__[name] = value + return + # set attribute on decoree + setattr(self.__dict__['_decoree'], name, value) diff --git a/colcon_python_project/hook_caller_decorator/setuptools.py b/colcon_python_project/hook_caller_decorator/setuptools.py new file mode 100644 index 0000000..87d67c4 --- /dev/null +++ b/colcon_python_project/hook_caller_decorator/setuptools.py @@ -0,0 +1,92 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from tempfile import TemporaryDirectory + +from colcon_core.plugin_system import satisfies_version +from colcon_python_project.hook_caller_decorator import GenericDecorator +from colcon_python_project.hook_caller_decorator \ + import HookCallerDecoratorExtensionPoint + + +class SetuptoolsHookCallerDecoratorExtension( + HookCallerDecoratorExtensionPoint +): + """Decorate a hook caller for a setuptools-based backend.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + HookCallerDecoratorExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + def decorate_hook_caller(self, *, hook_caller): # noqa: D102 + if hook_caller.backend_name not in ( + 'setuptools.build_meta', + 'setuptools.build_meta:__legacy__', + ): + return hook_caller + return SetuptoolsDecorator(hook_caller) + + +class _ScratchBuildBase(TemporaryDirectory): + + def __init__(self, config_settings): + self._config_settings = config_settings + super().__init__() + + def __enter__(self): + temp = super().__enter__() + self._config_settings.setdefault('--build-option', []) + self._config_settings['--build-option'] += [ + 'build', + f'--build-base={temp}', + ] + return temp + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + + +class _ScratchEggBase(TemporaryDirectory): + + def __init__(self, config_settings): + self._config_settings = config_settings + super().__init__() + + def __enter__(self): + temp = super().__enter__() + self._config_settings.setdefault('--build-option', []) + self._config_settings['--build-option'] += [ + 'egg_info', + f'--egg-base={temp}', + ] + return temp + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + + +class SetuptoolsDecorator(GenericDecorator): + """Enhance hooks to the setuptools build backend.""" + + async def build_wheel(self, **kwargs): # noqa: D102 + config_settings = kwargs.pop('config_settings', {}) + with ( + _ScratchEggBase(config_settings), + _ScratchBuildBase(config_settings), + ): + return await self._decoree.build_wheel( + config_settings=config_settings, **kwargs) + + async def get_requires_for_build_wheel(self, **kwargs): # noqa: D102 + config_settings = kwargs.pop('config_settings', {}) + with _ScratchEggBase(config_settings): + return await self._decoree.get_requires_for_build_wheel( + config_settings=config_settings, **kwargs) + + async def prepare_metadata_for_build_wheel(self, **kwargs): # noqa: D102 + config_settings = kwargs.pop('config_settings', {}) + with _ScratchEggBase(config_settings): + return await self._decoree.prepare_metadata_for_build_wheel( + config_settings=config_settings, **kwargs) diff --git a/setup.cfg b/setup.cfg index d580f86..f99ab24 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,10 +58,14 @@ filterwarnings = junit_suite_name = colcon-python-project [options.entry_points] +colcon_core.extension_point = + colcon_python_project.hook_caller_decorator = colcon_python_project.hook_caller_decorator:HookCallerDecoratorExtensionPoint colcon_core.package_augmentation = pep621 = colcon_python_project.package_augmentation.pep621:PEP621PackageAugmentation colcon_core.package_identification = pep621 = colcon_python_project.package_identification.pep621:PEP621PackageIdentification +colcon_python_project.hook_caller_decorator = + setuptools = colcon_python_project.hook_caller_decorator.setuptools:SetuptoolsHookCallerDecoratorExtension [flake8] import-order-style = google diff --git a/test/spell_check.words b/test/spell_check.words index d97d4f5..400517a 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -3,6 +3,8 @@ asyncio backend colcon contextlib +cottsay +decoree deps distinfo fdopen @@ -16,10 +18,13 @@ plugin pyproject pytest returncode +rtype scspell sdist setuptools +tempfile thomas +todo toml tomli tomllib From 7bf6c516965d3ebbe5a3dc6f2eda5f44e7257a0d Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 12:29:45 -0800 Subject: [PATCH 05/67] Implement mechanism for reading python METADATA files --- colcon_python_project/metadata.py | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 colcon_python_project/metadata.py diff --git a/colcon_python_project/metadata.py b/colcon_python_project/metadata.py new file mode 100644 index 0000000..26f4a9f --- /dev/null +++ b/colcon_python_project/metadata.py @@ -0,0 +1,40 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from email.parser import Parser +from pathlib import Path +from tempfile import TemporaryDirectory + +from colcon_python_project.hook_caller_decorator \ + import get_decorated_hook_caller + + +async def load_metadata(desc): + """ + Load metadata for a Python project using PEP 517. + + :param desc: The package descriptor + """ + hook_caller = get_decorated_hook_caller(desc) + with TemporaryDirectory() as md_dir: + md_name = await hook_caller.prepare_metadata_for_build_wheel( + metadata_directory=md_dir) + md_path = Path(md_dir) / md_name / 'METADATA' + with open(md_path) as f: + metadata = Parser().parse(f) + return metadata + + +async def load_and_cache_metadata(desc): + """ + Get the cached metadata for a package descriptor. + + If the metadata has not been loaded yet, load and cache it. + + :param desc: The package descriptor + """ + metadata = desc.metadata.get('python_project_metadata') + if metadata is None: + metadata = await load_metadata(desc) + desc.metadata['python_project_metadata'] = metadata + return metadata From 7ad21f50803150ce933f6494d61d2af825d468e1 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 12:32:21 -0800 Subject: [PATCH 06/67] Implement PEP 517 identification and augmentation --- .../package_augmentation/pep517.py | 97 +++++++++++++++++++ .../package_identification/pep517.py | 65 +++++++++++++ setup.cfg | 3 + stdeb.cfg | 2 +- test/spell_check.words | 2 + 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 colcon_python_project/package_augmentation/pep517.py create mode 100644 colcon_python_project/package_identification/pep517.py diff --git a/colcon_python_project/package_augmentation/pep517.py b/colcon_python_project/package_augmentation/pep517.py new file mode 100644 index 0000000..ccf201c --- /dev/null +++ b/colcon_python_project/package_augmentation/pep517.py @@ -0,0 +1,97 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import logging +from subprocess import CalledProcessError + +from colcon_core.logging import colcon_logger +from colcon_core.package_augmentation \ + import PackageAugmentationExtensionPoint +from colcon_core.package_augmentation.python \ + import create_dependency_descriptor +from colcon_core.plugin_system import satisfies_version +from colcon_core.subprocess import new_event_loop +from colcon_python_project.hook_caller_decorator \ + import get_decorated_hook_caller +from colcon_python_project.metadata import load_and_cache_metadata +from distlib.util import parse_requirement + +logger = colcon_logger.getChild(__name__) + +_TEST_EXTRAS = ( + "'test'", '"test"', + "'tests'", '"tests"', + "'testing'", '"testing"', +) + + +class PEP517PackageAugmentation(PackageAugmentationExtensionPoint): + """Augment Python packages with `pyproject.toml` using a build backend.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + # avoid debug message from asyncio when colcon uses debug log level + asyncio_logger = logging.getLogger('asyncio') + asyncio_logger.setLevel(logging.INFO) + + def augment_package( # noqa: D102 + self, desc, *, additional_argument_names=None + ): + if desc.type != 'python.project': + return + + loop = new_event_loop() + hook_caller = get_decorated_hook_caller(desc) + # TODO(cottsay): get_requires_for_build_editable + try: + deps = loop.run_until_complete( + hook_caller.get_requires_for_build_wheel()) + except CalledProcessError as e: + logger.warn( + f'An error occurred while reading metadata for {desc.name}:' + f" {e.stderr.strip().decode() or '(no output)'}") + return + desc.dependencies.setdefault('build', set()) + desc.dependencies['build'].update( + create_dependency_descriptor(d) for d in deps) + + try: + metadata = loop.run_until_complete( + load_and_cache_metadata(desc)) + except CalledProcessError as e: + logger.warn( + f'An error occurred while reading metadata for {desc.name}:' + f" {e.stderr.strip().decode() or '(no output)'}") + return + desc.dependencies.setdefault('run', set()) + desc.dependencies.setdefault('test', set()) + for raw_req in metadata.get_all('Requires-Dist', ()): + req = parse_requirement(raw_req) + if req.marker: + if ( + req.marker['lhs'] == 'extra' and + req.marker['op'] == '==' and + req.marker['rhs'] in _TEST_EXTRAS + ): + desc.dependencies['test'].add( + create_dependency_descriptor(raw_req)) + else: + desc.dependencies['run'].add( + create_dependency_descriptor(raw_req)) + + if not desc.metadata.get('version'): + desc.metadata['version'] = metadata['Version'] + + maintainers = metadata.get('Maintainer-email') + if not maintainers: + maintainers = metadata.get('Author-email') + if maintainers: + desc.metadata.setdefault('maintainers', []) + for maintainer in maintainers.split(','): + rfc822 = maintainer.strip() + if rfc822 not in desc.metadata['maintainers']: + desc.metadata['maintainers'].append(rfc822) diff --git a/colcon_python_project/package_identification/pep517.py b/colcon_python_project/package_identification/pep517.py new file mode 100644 index 0000000..393923a --- /dev/null +++ b/colcon_python_project/package_identification/pep517.py @@ -0,0 +1,65 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import logging +from subprocess import CalledProcessError + +from colcon_core.logging import colcon_logger +from colcon_core.package_identification \ + import PackageIdentificationExtensionPoint +from colcon_core.plugin_system import satisfies_version +from colcon_core.subprocess import new_event_loop +from colcon_python_project.metadata import load_and_cache_metadata +from colcon_python_project.spec import SPEC_NAME + +logger = colcon_logger.getChild(__name__) + + +class PEP517PackageIdentification(PackageIdentificationExtensionPoint): + """ + Identify Python packages with `pyproject.toml` using the build backend. + + This mechanism is very slow compared with other identification extensions. + It should be able to function with any PEP 517 compliant build backend, + but those backends which see widespread use should implement a more + efficient identification extension. + """ + + # the priority needs to be higher than the extensions identifying packages + # using setup.py and/or setup.cfg files but lower than other more efficient + # PEP 517 compliant identification extensions. + PRIORITY = 110 + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageIdentificationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + # avoid debug message from asyncio when colcon uses debug log level + asyncio_logger = logging.getLogger('asyncio') + asyncio_logger.setLevel(logging.INFO) + + def identify(self, desc): # noqa: D102 + if desc.type is not None and desc.type != 'python.project': + return + + spec_file = desc.path / SPEC_NAME + if not spec_file.is_file(): + return + + loop = new_event_loop() + try: + metadata = loop.run_until_complete( + load_and_cache_metadata(desc)) + except CalledProcessError as e: + logger.warn( + f'An error occurred while reading metadata for {desc.path}:' + f" {e.stderr.strip().decode() or '(no output)'}") + return + name = metadata.get('Name') + if not name: + return + + desc.name = name + desc.type = 'python.project' diff --git a/setup.cfg b/setup.cfg index f99ab24..48776e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ keywords = colcon [options] install_requires = colcon-core + distlib tomli>=1.0.0;python_version < "3.11" packages = find: zip_safe = true @@ -61,8 +62,10 @@ junit_suite_name = colcon-python-project colcon_core.extension_point = colcon_python_project.hook_caller_decorator = colcon_python_project.hook_caller_decorator:HookCallerDecoratorExtensionPoint colcon_core.package_augmentation = + pep517 = colcon_python_project.package_augmentation.pep517:PEP517PackageAugmentation pep621 = colcon_python_project.package_augmentation.pep621:PEP621PackageAugmentation colcon_core.package_identification = + pep517 = colcon_python_project.package_identification.pep517:PEP517PackageIdentification pep621 = colcon_python_project.package_identification.pep621:PEP621PackageIdentification colcon_python_project.hook_caller_decorator = setuptools = colcon_python_project.hook_caller_decorator.setuptools:SetuptoolsHookCallerDecoratorExtension diff --git a/stdeb.cfg b/stdeb.cfg index 052cfe9..551e2ca 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,5 +1,5 @@ [colcon-python-project] No-Python2: -Depends3: python3-colcon-core, python3-tomli (>= 1.0.0) +Depends3: python3-colcon-core, python3-distlib, python3-tomli (>= 1.0.0) Suite: jammy bullseye X-Python3-Version: >= 3.6 diff --git a/test/spell_check.words b/test/spell_check.words index 400517a..a763b63 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -1,12 +1,14 @@ apache asyncio backend +backends colcon contextlib cottsay decoree deps distinfo +distlib fdopen functools importlib From 772a78e29ed068cca812b9bcc6986b670cf9c4a5 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 12:28:06 -0800 Subject: [PATCH 07/67] Implement basic mechanism for installing Python wheels --- colcon_python_project/wheel.py | 88 ++++++++++++++++++++++++++++++++++ test/spell_check.words | 7 +++ 2 files changed, 95 insertions(+) create mode 100644 colcon_python_project/wheel.py diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py new file mode 100644 index 0000000..f756d05 --- /dev/null +++ b/colcon_python_project/wheel.py @@ -0,0 +1,88 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from configparser import ConfigParser +from email import message_from_binary_file +from io import TextIOWrapper +from pathlib import Path +import warnings +from zipfile import ZIP_DEFLATED +from zipfile import ZipFile + +from colcon_core.python_install_path import get_python_install_path +from distlib.scripts import ScriptMaker + + +def _get_install_path(key, install_base): + return get_python_install_path(key, {'base': str(install_base)}) + + +def install_wheel(wheel_path, install_base): + """ + Install a wheel file under the given installation base directory. + + :param wheel_path: Path to the wheel file to be installed. + :param install_base: Path to the base directory to install under. + """ + wheel_name = wheel_path.name.split('-') + if len(wheel_name) not in (5, 6): + raise RuntimeError('Invalid wheel file name') + distribution, version = wheel_name[:2] + dist_info_dir = f'{distribution}-{version}.dist-info/' + data_dir = f'{distribution}-{version}.data/' + wheel_file = dist_info_dir + 'WHEEL' + record_file = dist_info_dir + 'RECORD' + entry_points_file = dist_info_dir + 'entry_points.txt' + + with ZipFile( + wheel_path, mode='r', compression=ZIP_DEFLATED, allowZip64=True + ) as wf: + with wf.open(wheel_file) as wf_mf: + wheel_metadata = message_from_binary_file(wf_mf) + + wheel_version = wheel_metadata.get('Wheel-Version', '').split('.') + if len(wheel_version) < 2 or wheel_version[0] != '1': + raise RuntimeError('Wheel file is not supported') + elif wheel_version[1] != '0': + warnings.warn('Wheel format is newer than supported version') + + if wheel_metadata.get('Root-Is-Purelib') in ('true',): + libdir = _get_install_path('purelib', install_base) + else: + libdir = _get_install_path('platlib', install_base) + + records = [] + with wf.open(record_file) as wf_rec_bin: + with TextIOWrapper(wf_rec_bin) as wf_rec: + for line in wf_rec: + if ',' in line: + records.append(line.split(',')) + + for record in records: + if record[0] == record_file: + continue + elif not record[0].startswith(data_dir): + wf.extract(record[0], libdir) + continue + + _, key, subpath = record[0].split('/', 2) + target = Path(_get_install_path(key, install_base)) + target /= subpath + wf.extract(record[0], str(target)) + record[0] = target.relative_to(libdir) + + if entry_points_file in wf.namelist(): + ep = ConfigParser() + with wf.open(entry_points_file) as wf_ep_bin: + with TextIOWrapper(wf_ep_bin) as wf_ep: + ep.read_file(wf_ep) + if ep.has_section('console_scripts'): + script_dir = _get_install_path('scripts', install_base) + sm = ScriptMaker(None, script_dir) + sm.clobber = True + sm.variants = {''} + specs = [ + '%s = %s' % pair + for pair in ep.items('console_scripts') + ] + sm.make_multiple(specs) diff --git a/test/spell_check.words b/test/spell_check.words index a763b63..e565e1d 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -3,6 +3,7 @@ asyncio backend backends colcon +configparser contextlib cottsay decoree @@ -13,10 +14,14 @@ fdopen functools importlib iterdir +libdir +namelist noqa partialmethod pathlib +platlib plugin +purelib pyproject pytest returncode @@ -24,6 +29,7 @@ rtype scspell sdist setuptools +subpath tempfile thomas todo @@ -31,3 +37,4 @@ toml tomli tomllib traceback +zipfile From 89a83c08559dd35ccf48736f5a416b2e6bb04e5d Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 12:39:10 -0800 Subject: [PATCH 08/67] Implement PEP 517 build task --- colcon_python_project/task/__init__.py | 0 colcon_python_project/task/python/__init__.py | 0 .../task/python/project/__init__.py | 0 .../task/python/project/build.py | 72 +++++++++++++++++++ setup.cfg | 2 + test/spell_check.words | 1 + 6 files changed, 75 insertions(+) create mode 100644 colcon_python_project/task/__init__.py create mode 100644 colcon_python_project/task/python/__init__.py create mode 100644 colcon_python_project/task/python/project/__init__.py create mode 100644 colcon_python_project/task/python/project/build.py diff --git a/colcon_python_project/task/__init__.py b/colcon_python_project/task/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/task/python/__init__.py b/colcon_python_project/task/python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/task/python/project/__init__.py b/colcon_python_project/task/python/project/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/task/python/project/build.py b/colcon_python_project/task/python/project/build.py new file mode 100644 index 0000000..e60c293 --- /dev/null +++ b/colcon_python_project/task/python/project/build.py @@ -0,0 +1,72 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import logging +from pathlib import Path +from subprocess import CalledProcessError + +from colcon_core.environment import create_environment_hooks +from colcon_core.environment import create_environment_scripts +from colcon_core.event.output import StderrLine +from colcon_core.event.output import StdoutLine +from colcon_core.logging import colcon_logger +from colcon_core.plugin_system import satisfies_version +from colcon_core.shell import get_command_environment +from colcon_core.task import TaskExtensionPoint +from colcon_python_project.hook_caller_decorator \ + import get_decorated_hook_caller +from colcon_python_project.wheel import install_wheel + +logger = colcon_logger.getChild(__name__) + + +class PythonProjectBuildTask(TaskExtensionPoint): + """Build Python project packages.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version(TaskExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + # avoid debug message from asyncio when colcon uses debug log level + asyncio_logger = logging.getLogger('asyncio') + asyncio_logger.setLevel(logging.INFO) + + distutils_logger = logging.getLogger('distlib.util') + distutils_logger.setLevel(logging.WARN) + + async def build(self, *, additional_hooks=None): # noqa: D102 + pkg = self.context.pkg + args = self.context.args + + logger.info(f"Building Python project in '{args.path}'") + + env = await get_command_environment( + 'python_project', args.build_base, self.context.dependencies) + + hook_caller = get_decorated_hook_caller( + pkg, env=env, stdout_callback=self._stdout_callback, + stderr_callback=self._stderr_callback) + + wheel_directory = Path(args.build_base) / 'wheel' + if not wheel_directory.is_dir(): + wheel_directory.mkdir() + try: + if args.symlink_install: + logger.warn(f'Symlink install is not supported by {__name__}') + wheel_name = await hook_caller.build_wheel( + wheel_directory=wheel_directory) + except CalledProcessError as e: + return e.returncode + + wheel_path = wheel_directory / wheel_name + install_wheel(wheel_path, args.install_base) + + hooks = create_environment_hooks(args.install_base, pkg.name) + create_environment_scripts( + pkg, args, default_hooks=hooks, additional_hooks=additional_hooks) + + def _stdout_callback(self, line): + self.context.put_event_into_queue(StdoutLine(line)) + + def _stderr_callback(self, line): + self.context.put_event_into_queue(StderrLine(line)) diff --git a/setup.cfg b/setup.cfg index 48776e9..ab0a341 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,6 +67,8 @@ colcon_core.package_augmentation = colcon_core.package_identification = pep517 = colcon_python_project.package_identification.pep517:PEP517PackageIdentification pep621 = colcon_python_project.package_identification.pep621:PEP621PackageIdentification +colcon_core.task.build = + python.project = colcon_python_project.task.python.project.build:PythonProjectBuildTask colcon_python_project.hook_caller_decorator = setuptools = colcon_python_project.hook_caller_decorator.setuptools:SetuptoolsHookCallerDecoratorExtension diff --git a/test/spell_check.words b/test/spell_check.words index e565e1d..1ed0391 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -30,6 +30,7 @@ scspell sdist setuptools subpath +symlink tempfile thomas todo From b41a6d5223af1b4bd20dd28eb9f99be2bc043a02 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 12:40:29 -0800 Subject: [PATCH 09/67] Implement test task for python.project packages --- colcon_python_project/task/python/project/test.py | 11 +++++++++++ setup.cfg | 2 ++ 2 files changed, 13 insertions(+) create mode 100644 colcon_python_project/task/python/project/test.py diff --git a/colcon_python_project/task/python/project/test.py b/colcon_python_project/task/python/project/test.py new file mode 100644 index 0000000..73104ca --- /dev/null +++ b/colcon_python_project/task/python/project/test.py @@ -0,0 +1,11 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_core.task.python.test import PythonTestTask + + +class PythonProjectTestTask(PythonTestTask): + """Test Python project packages.""" + + def add_arguments(self, *, parser): # noqa: D102 + pass diff --git a/setup.cfg b/setup.cfg index ab0a341..dee3823 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,6 +69,8 @@ colcon_core.package_identification = pep621 = colcon_python_project.package_identification.pep621:PEP621PackageIdentification colcon_core.task.build = python.project = colcon_python_project.task.python.project.build:PythonProjectBuildTask +colcon_core.task.test = + python.project = colcon_python_project.task.python.project.test:PythonProjectTestTask colcon_python_project.hook_caller_decorator = setuptools = colcon_python_project.hook_caller_decorator.setuptools:SetuptoolsHookCallerDecoratorExtension From ca4305467914989d9406609fa3b5aa122a6338c2 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Nov 2022 12:41:42 -0800 Subject: [PATCH 10/67] Update README.rst to reflect current status --- README.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.rst b/README.rst index 00b6513..4258156 100644 --- a/README.rst +++ b/README.rst @@ -5,3 +5,14 @@ Extensions for `colcon-core `_ to work wi **:warning: This is a prototype package, and isn't ready for widespread use.** **Please continue using the Python package support in** ``colcon-core`` **and** ``colcon-python-setup-py`` **until these extensions are ready.** + +TODO +---- +* Install RECORD file +* Graceful and informational error handling +* Finish PEP 660 (symlink) installs + +Idiosyncrasies +-------------- +* For setuptools-based packages, setuptools (< 64.0.0) will leave build artifacts in the source directory. +* For poetry-based packages, dependencies expressed in groups (including 'test') are not discovered (use 'test' extra as a workaround). From bd3522013ac89d4ce96af1b07370251548944d24 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 1 Dec 2022 11:14:23 -0800 Subject: [PATCH 11/67] Add support for in-tree build backends --- colcon_python_project/hook_caller.py | 8 ++++++++ test/spell_check.words | 1 + 2 files changed, 9 insertions(+) diff --git a/colcon_python_project/hook_caller.py b/colcon_python_project/hook_caller.py index 6237264..b5131df 100644 --- a/colcon_python_project/hook_caller.py +++ b/colcon_python_project/hook_caller.py @@ -102,6 +102,14 @@ def get_hook_caller(desc, **kwargs): :param desc: The package descriptor """ spec = load_and_cache_spec(desc) + backend_path = spec['build-system'].get('backend-path') + if backend_path: + # TODO(cottsay): This isn't *technically* the beginning of sys.path + # as PEP 517 calls for, but it's pretty darn close. + kwargs['env'] = dict(kwargs.get('env', os.environ)) + pythonpath = kwargs['env'].get('PYTHONPATH', '') + kwargs['env']['PYTHONPATH'] = os.pathsep.join( + backend_path + ([pythonpath] if pythonpath else [])) return AsyncHookCaller( spec['build-system']['build-backend'], project_path=desc.path, **kwargs) diff --git a/test/spell_check.words b/test/spell_check.words index 1ed0391..5d5550e 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -24,6 +24,7 @@ plugin purelib pyproject pytest +pythonpath returncode rtype scspell From 04345c0fadfb355b35e328a8e96fa38fd4d6ff1d Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 1 Dec 2022 11:23:40 -0800 Subject: [PATCH 12/67] Add support for setuptools script directory override --- .../task/python/project/build.py | 33 ++++++++++++++++++- colcon_python_project/wheel.py | 9 +++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/colcon_python_project/task/python/project/build.py b/colcon_python_project/task/python/project/build.py index e60c293..afa4224 100644 --- a/colcon_python_project/task/python/project/build.py +++ b/colcon_python_project/task/python/project/build.py @@ -1,6 +1,7 @@ # Copyright 2022 Open Source Robotics Foundation, Inc. # Licensed under the Apache License, Version 2.0 +from configparser import ConfigParser import logging from pathlib import Path from subprocess import CalledProcessError @@ -40,6 +41,34 @@ async def build(self, *, additional_hooks=None): # noqa: D102 logger.info(f"Building Python project in '{args.path}'") + script_dir_override = None + setup_cfg_path = pkg.path / 'setup.cfg' + if setup_cfg_path.is_file(): + parser = ConfigParser() + with setup_cfg_path.open() as f: + parser.read_file(f) + if args.symlink_install: + script_dir_override = parser.get( + 'develop', 'script-dir', fallback=None) + if not script_dir_override: + script_dir_override = parser.get( + 'develop', 'script_dir', fallback=None) + else: + script_dir_override = parser.get( + 'install', 'install-scripts', fallback=None) + if not script_dir_override: + script_dir_override = parser.get( + 'install', 'install_scripts', fallback=None) + + # Resolve setuptools-specific syntax + if script_dir_override: + _override = Path(args.install_base) + for part in Path(script_dir_override).parts: + if part == '$base': + part = args.install_base + _override /= part + script_dir_override = _override + env = await get_command_environment( 'python_project', args.build_base, self.context.dependencies) @@ -59,7 +88,9 @@ async def build(self, *, additional_hooks=None): # noqa: D102 return e.returncode wheel_path = wheel_directory / wheel_name - install_wheel(wheel_path, args.install_base) + install_wheel( + wheel_path, args.install_base, + script_dir_override=script_dir_override) hooks = create_environment_hooks(args.install_base, pkg.name) create_environment_scripts( diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index f756d05..49a63a9 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -17,12 +17,14 @@ def _get_install_path(key, install_base): return get_python_install_path(key, {'base': str(install_base)}) -def install_wheel(wheel_path, install_base): +def install_wheel(wheel_path, install_base, script_dir_override=None): """ Install a wheel file under the given installation base directory. :param wheel_path: Path to the wheel file to be installed. :param install_base: Path to the base directory to install under. + :param script_dir_override: Override the default script install + directory """ wheel_name = wheel_path.name.split('-') if len(wheel_name) not in (5, 6): @@ -77,7 +79,10 @@ def install_wheel(wheel_path, install_base): with TextIOWrapper(wf_ep_bin) as wf_ep: ep.read_file(wf_ep) if ep.has_section('console_scripts'): - script_dir = _get_install_path('scripts', install_base) + if script_dir_override: + script_dir = install_base / script_dir_override + else: + script_dir = _get_install_path('scripts', install_base) sm = ScriptMaker(None, script_dir) sm.clobber = True sm.variants = {''} From 74968bc74235e2df869034be84243d514d75372f Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 1 Dec 2022 12:01:18 -0800 Subject: [PATCH 13/67] Fix installation of data files from wheels --- colcon_python_project/wheel.py | 13 +++++++++++-- test/spell_check.words | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index 49a63a9..7409930 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -4,7 +4,9 @@ from configparser import ConfigParser from email import message_from_binary_file from io import TextIOWrapper +import os.path from pathlib import Path +import shutil import warnings from zipfile import ZIP_DEFLATED from zipfile import ZipFile @@ -70,8 +72,11 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): _, key, subpath = record[0].split('/', 2) target = Path(_get_install_path(key, install_base)) target /= subpath - wf.extract(record[0], str(target)) - record[0] = target.relative_to(libdir) + target.parent.mkdir(parents=True, exist_ok=True) + with wf.open(record[0]) as fsrc: + with target.open('wb') as fdst: + shutil.copyfileobj(fsrc, fdst) + record[0] = os.path.relpath(target, start=libdir) if entry_points_file in wf.namelist(): ep = ConfigParser() @@ -91,3 +96,7 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): for pair in ep.items('console_scripts') ] sm.make_multiple(specs) + + # TODO(cottsay): Add scripts to records + + # TODO(cottsay): Write out records to RECORDS file diff --git a/test/spell_check.words b/test/spell_check.words index 5d5550e..5fc060f 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -5,12 +5,15 @@ backends colcon configparser contextlib +copyfileobj cottsay decoree deps distinfo distlib fdopen +fdst +fsrc functools importlib iterdir @@ -25,6 +28,7 @@ purelib pyproject pytest pythonpath +relpath returncode rtype scspell From 65c07f9dc5d562358411a76acbf2fbf5f09c3063 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 1 Dec 2022 15:00:51 -0800 Subject: [PATCH 14/67] More robust wheel directory creation --- colcon_python_project/task/python/project/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/colcon_python_project/task/python/project/build.py b/colcon_python_project/task/python/project/build.py index afa4224..f348c28 100644 --- a/colcon_python_project/task/python/project/build.py +++ b/colcon_python_project/task/python/project/build.py @@ -77,8 +77,7 @@ async def build(self, *, additional_hooks=None): # noqa: D102 stderr_callback=self._stderr_callback) wheel_directory = Path(args.build_base) / 'wheel' - if not wheel_directory.is_dir(): - wheel_directory.mkdir() + wheel_directory.mkdir(parents=True, exist_ok=True) try: if args.symlink_install: logger.warn(f'Symlink install is not supported by {__name__}') From ab5b6e33582d6da5e97b1b0208cca399fc4dedde Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 1 Dec 2022 15:01:28 -0800 Subject: [PATCH 15/67] Add a PEP 517 identification fallback for legacy setuptools --- .../package_identification/pep517.py | 7 ++-- .../pep517_setuptools_fallback.py | 41 +++++++++++++++++++ setup.cfg | 1 + test/spell_check.words | 1 + 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 colcon_python_project/package_identification/pep517_setuptools_fallback.py diff --git a/colcon_python_project/package_identification/pep517.py b/colcon_python_project/package_identification/pep517.py index 393923a..58c83c1 100644 --- a/colcon_python_project/package_identification/pep517.py +++ b/colcon_python_project/package_identification/pep517.py @@ -44,9 +44,10 @@ def identify(self, desc): # noqa: D102 if desc.type is not None and desc.type != 'python.project': return - spec_file = desc.path / SPEC_NAME - if not spec_file.is_file(): - return + if desc.type is None: + spec_file = desc.path / SPEC_NAME + if not spec_file.is_file(): + return loop = new_event_loop() try: diff --git a/colcon_python_project/package_identification/pep517_setuptools_fallback.py b/colcon_python_project/package_identification/pep517_setuptools_fallback.py new file mode 100644 index 0000000..422fa5d --- /dev/null +++ b/colcon_python_project/package_identification/pep517_setuptools_fallback.py @@ -0,0 +1,41 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_python_project.package_identification.pep517 \ + import PEP517PackageIdentification + + +class PEP517SetuptoolsFallbackPackageIdentification( + PEP517PackageIdentification +): + """ + Identify legacy setuptools packages using PEP 517. + + To use this extension, disable the ``python`` and ``python_setup_py`` + identification extensions using this environment variable: + ``COLCON_EXTENSION_BLOCKLIST=colcon_core.package_identification.python:colcon_core.package_identification.python_setup_py`` + """ + + # the priority needs to be lower than all other Python package + # identification extensions. + PRIORITY = 80 + + def identify(self, desc): # noqa: D102 + if desc.type is not None and desc.type != 'python.project': + return + + setup_cfg_file = desc.path / 'setup.cfg' + setup_py_file = desc.path / 'setup.py' + + if not setup_cfg_file.is_file() and not setup_py_file.is_file(): + return + + if desc.type is not None: + return super().identify(desc) + + desc.type = 'python.project' + try: + return super().identify(desc) + finally: + if not desc.name: + desc.type = None diff --git a/setup.cfg b/setup.cfg index dee3823..40188da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,6 +66,7 @@ colcon_core.package_augmentation = pep621 = colcon_python_project.package_augmentation.pep621:PEP621PackageAugmentation colcon_core.package_identification = pep517 = colcon_python_project.package_identification.pep517:PEP517PackageIdentification + pep517_setuptools_fallback = colcon_python_project.package_identification.pep517_setuptools_fallback:PEP517SetuptoolsFallbackPackageIdentification pep621 = colcon_python_project.package_identification.pep621:PEP621PackageIdentification colcon_core.task.build = python.project = colcon_python_project.task.python.project.build:PythonProjectBuildTask diff --git a/test/spell_check.words b/test/spell_check.words index 5fc060f..36cf924 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -2,6 +2,7 @@ apache asyncio backend backends +blocklist colcon configparser contextlib From 98f164dc4b4fbc1bec81265c1840a0f95c347fb2 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 1 Dec 2022 15:58:41 -0800 Subject: [PATCH 16/67] Implement RECORD file installation for wheels --- README.rst | 1 - colcon_python_project/wheel.py | 12 ++++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 4258156..c072638 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,6 @@ Extensions for `colcon-core `_ to work wi TODO ---- -* Install RECORD file * Graceful and informational error handling * Finish PEP 660 (symlink) installs diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index 7409930..6b70fbb 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -60,7 +60,7 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): with TextIOWrapper(wf_rec_bin) as wf_rec: for line in wf_rec: if ',' in line: - records.append(line.split(',')) + records.append(line.strip().split(',')) for record in records: if record[0] == record_file: @@ -95,8 +95,12 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): '%s = %s' % pair for pair in ep.items('console_scripts') ] - sm.make_multiple(specs) + scripts_made = sm.make_multiple(specs) - # TODO(cottsay): Add scripts to records + records += [ + (Path(os.path.relpath(s, libdir)).as_posix(), '', '') + for s in scripts_made + ] - # TODO(cottsay): Write out records to RECORDS file + with (libdir / record_file).open('w') as f: + f.writelines(','.join(rec) + '\n' for rec in records) From 03572cc044a5f75f22a2ae01d5bf19cf725d6804 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 1 Dec 2022 17:25:53 -0800 Subject: [PATCH 17/67] Add missing test dependency on pytest-asyncio --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 40188da..f658b86 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,7 @@ test = pep8-naming pylint pytest + pytest-asyncio pytest-cov scspell3k>=2.2 setuptools>=40.8.0 From 977e9c5116bd4e6bf7052738cc569ec76a8f5de1 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 1 Dec 2022 17:26:12 -0800 Subject: [PATCH 18/67] Forward subprocess stderr in hook tests --- test/test_hook_caller_setuptools.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/test/test_hook_caller_setuptools.py b/test/test_hook_caller_setuptools.py index dbb2fa9..b6bb34b 100644 --- a/test/test_hook_caller_setuptools.py +++ b/test/test_hook_caller_setuptools.py @@ -1,6 +1,8 @@ # Copyright 2022 Open Source Robotics Foundation, Inc. # Licensed under the Apache License, Version 2.0 +import sys + from colcon_python_project.hook_caller import AsyncHookCaller import pytest @@ -37,7 +39,8 @@ def mock_project(tmp_path): @pytest.mark.asyncio async def test_build_wheel(mock_project, tmp_path): hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project) + _BACKEND_NAME, project_path=mock_project, + stderr_callback=sys.stderr.buffer.write) wheel = await hook_caller.build_wheel(wheel_directory=str(tmp_path)) assert isinstance(wheel, str) assert (tmp_path / wheel).is_file() @@ -46,7 +49,8 @@ async def test_build_wheel(mock_project, tmp_path): @pytest.mark.asyncio async def test_build_sdist(mock_project, tmp_path): hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project) + _BACKEND_NAME, project_path=mock_project, + stderr_callback=sys.stderr.buffer.write) sdist = await hook_caller.build_sdist(sdist_directory=str(tmp_path)) assert isinstance(sdist, str) assert (tmp_path / sdist).is_file() @@ -55,7 +59,8 @@ async def test_build_sdist(mock_project, tmp_path): @pytest.mark.asyncio async def test_get_requires_for_build_wheel(mock_project): hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project) + _BACKEND_NAME, project_path=mock_project, + stderr_callback=sys.stderr.buffer.write) requires = await hook_caller.get_requires_for_build_wheel() assert isinstance(requires, list) @@ -63,7 +68,8 @@ async def test_get_requires_for_build_wheel(mock_project): @pytest.mark.asyncio async def test_prepare_metadata_for_build_wheel(mock_project, tmp_path): hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project) + _BACKEND_NAME, project_path=mock_project, + stderr_callback=sys.stderr.buffer.write) distinfo = await hook_caller.prepare_metadata_for_build_wheel( metadata_directory=str(tmp_path)) assert isinstance(distinfo, str) @@ -73,7 +79,8 @@ async def test_prepare_metadata_for_build_wheel(mock_project, tmp_path): @pytest.mark.asyncio async def test_get_requires_for_build_sdist(mock_project): hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project) + _BACKEND_NAME, project_path=mock_project, + stderr_callback=sys.stderr.buffer.write) requires = await hook_caller.get_requires_for_build_sdist() assert isinstance(requires, list) @@ -82,7 +89,8 @@ async def test_get_requires_for_build_sdist(mock_project): @pytest.mark.skip(reason='Insufficient setuptools version') async def test_build_editable(mock_project, tmp_path): hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project) + _BACKEND_NAME, project_path=mock_project, + stderr_callback=sys.stderr.buffer.write) wheel = await hook_caller.build_editable( wheel_directory=str(tmp_path)) assert isinstance(wheel, str) @@ -93,7 +101,8 @@ async def test_build_editable(mock_project, tmp_path): @pytest.mark.skip(reason='Insufficient setuptools version') async def test_get_requires_for_build_editable(mock_project): hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project) + _BACKEND_NAME, project_path=mock_project, + stderr_callback=sys.stderr.buffer.write) requires = await hook_caller.get_requires_for_build_editable() assert isinstance(requires, list) @@ -102,7 +111,8 @@ async def test_get_requires_for_build_editable(mock_project): @pytest.mark.skip(reason='Insufficient setuptools version') async def test_prepare_metadata_for_build_editable(mock_project, tmp_path): hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project) + _BACKEND_NAME, project_path=mock_project, + stderr_callback=sys.stderr.buffer.write) dist_info = await hook_caller.prepare_metadata_for_build_editable( metadata_directory=str(tmp_path)) assert isinstance(dist_info, str) From 4614fa9be5f7a68cc0b13921981ef4bb29820567 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 1 Dec 2022 17:27:36 -0800 Subject: [PATCH 19/67] Forward handles instead of fds on Windows --- colcon_python_project/_call_hook.py | 7 +++++++ colcon_python_project/hook_caller.py | 18 +++++++++++++++--- test/spell_check.words | 1 + 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/colcon_python_project/_call_hook.py b/colcon_python_project/_call_hook.py index 0bfc8de..e9e86aa 100644 --- a/colcon_python_project/_call_hook.py +++ b/colcon_python_project/_call_hook.py @@ -9,6 +9,13 @@ if __name__ == '__main__': backend_name, hook_name, child_in, child_out = sys.argv[1:] + try: + import msvcrt + except ImportError: + pass + else: + child_in = msvcrt.open_osfhandle(int(child_in), os.O_RDONLY) + child_out = msvcrt.open_osfhandle(int(child_out), 0) if ':' in backend_name: backend_module_name, backend_object_name = backend_name.split(':', 2) backend_module = import_module(backend_module_name) diff --git a/colcon_python_project/hook_caller.py b/colcon_python_project/hook_caller.py index b5131df..2b938df 100644 --- a/colcon_python_project/hook_caller.py +++ b/colcon_python_project/hook_caller.py @@ -15,9 +15,21 @@ class _SubprocessTransport(AbstractContextManager): def __enter__(self): self.child_in, self.parent_out = os.pipe() - os.set_inheritable(self.child_in, True) self.parent_in, self.child_out = os.pipe() - os.set_inheritable(self.child_out, True) + + try: + import msvcrt + except ImportError: + os.set_inheritable(self.child_in, True) + self.pass_in = self.child_in + os.set_inheritable(self.child_out, True) + self.pass_out = self.child_out + else: + self.pass_in = msvcrt.get_osfhandle(self.child_in) + os.set_handle_inheritable(self.pass_in, True) + self.pass_out = msvcrt.get_osfhandle(self.child_out) + os.set_handle_inheritable(self.pass_out, True) + return self def __exit__(self, exc_type, exc_value, traceback): @@ -64,7 +76,7 @@ async def call_hook(self, hook_name, **kwargs): args = [ sys.executable, '-m', 'colcon_python_project._call_hook', self._backend_name, hook_name, - str(transport.child_in), str(transport.child_out)] + str(transport.pass_in), str(transport.pass_out)] with os.fdopen(os.dup(transport.parent_out), 'wb') as f: pickle.dump(kwargs, f) have_callbacks = self._stdout_callback or self._stderr_callback diff --git a/test/spell_check.words b/test/spell_check.words index 36cf924..ebb5286 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -21,6 +21,7 @@ iterdir libdir namelist noqa +osfhandle partialmethod pathlib platlib From 62e608d2e24e2b75f47e5b2bab0fddb1b3076edc Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 5 Dec 2022 14:45:15 -0800 Subject: [PATCH 20/67] Add 'INSTALLER' file to dist-info --- colcon_python_project/wheel.py | 27 +++++++++++++++++++++++++++ test/spell_check.words | 3 +++ 2 files changed, 30 insertions(+) diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index 6b70fbb..dd0d468 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -1,8 +1,10 @@ # Copyright 2022 Open Source Robotics Foundation, Inc. # Licensed under the Apache License, Version 2.0 +from base64 import urlsafe_b64encode from configparser import ConfigParser from email import message_from_binary_file +from hashlib import sha256 from io import TextIOWrapper import os.path from pathlib import Path @@ -19,6 +21,26 @@ def _get_install_path(key, install_base): return get_python_install_path(key, {'base': str(install_base)}) +def write_and_record(libdir, path, lines): + """ + Write file content to disk and compute a fully-qualified RECORD entry. + + :param libdir: The library directory where the package is installed. + :param path: Path to the file to be written, either absolute or + relative to the library directory. + :param lines: Enumerable of lines of text to be written. + :returns: Three-element tuple constituting the file's RECORD entry. + """ + path = libdir / path + raw = (os.linesep.join(lines) + os.linesep).encode() + digest = urlsafe_b64encode(sha256(raw).digest()).rstrip(b'=').decode() + path.write_bytes(raw) + return ( + Path(os.path.relpath(path, libdir)).as_posix(), + f'sha256={digest}', + f'{len(raw)}') + + def install_wheel(wheel_path, install_base, script_dir_override=None): """ Install a wheel file under the given installation base directory. @@ -78,6 +100,11 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): shutil.copyfileobj(fsrc, fdst) record[0] = os.path.relpath(target, start=libdir) + records.append(write_and_record( + libdir, + dist_info_dir + 'INSTALLER', + ('colcon-python-project',))) + if entry_points_file in wf.namelist(): ep = ConfigParser() with wf.open(entry_points_file) as wf_ep_bin: diff --git a/test/spell_check.words b/test/spell_check.words index ebb5286..922e194 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -16,6 +16,7 @@ fdopen fdst fsrc functools +hashlib importlib iterdir libdir @@ -32,6 +33,7 @@ pytest pythonpath relpath returncode +rstrip rtype scspell sdist @@ -45,4 +47,5 @@ toml tomli tomllib traceback +urlsafe zipfile From 986ad194164b6d099aa8815200f6c82ec2b4fc06 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 5 Dec 2022 14:13:17 -0800 Subject: [PATCH 21/67] Add hooks and scripts to RECORD file --- .../task/python/project/build.py | 21 +++++++++++++++++-- colcon_python_project/wheel.py | 3 +++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/colcon_python_project/task/python/project/build.py b/colcon_python_project/task/python/project/build.py index f348c28..ae50a23 100644 --- a/colcon_python_project/task/python/project/build.py +++ b/colcon_python_project/task/python/project/build.py @@ -3,6 +3,7 @@ from configparser import ConfigParser import logging +import os.path from pathlib import Path from subprocess import CalledProcessError @@ -87,13 +88,29 @@ async def build(self, *, additional_hooks=None): # noqa: D102 return e.returncode wheel_path = wheel_directory / wheel_name - install_wheel( + record_file = install_wheel( wheel_path, args.install_base, script_dir_override=script_dir_override) + libdir = record_file.parent.parent + records = [] + hooks = create_environment_hooks(args.install_base, pkg.name) - create_environment_scripts( + records += [ + (Path(os.path.relpath(hook, libdir)).as_posix(), '', '') + for hook in hooks + ] + + scripts = create_environment_scripts( pkg, args, default_hooks=hooks, additional_hooks=additional_hooks) + if scripts: + records += [ + (Path(os.path.relpath(script, libdir)).as_posix(), '', '') + for script in scripts + ] + + with record_file.open('a') as f: + f.writelines(','.join(rec) + '\n' for rec in records) def _stdout_callback(self, line): self.context.put_event_into_queue(StdoutLine(line)) diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index dd0d468..5da3aff 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -49,6 +49,7 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): :param install_base: Path to the base directory to install under. :param script_dir_override: Override the default script install directory + :returns: Path to the installed RECORD file """ wheel_name = wheel_path.name.split('-') if len(wheel_name) not in (5, 6): @@ -131,3 +132,5 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): with (libdir / record_file).open('w') as f: f.writelines(','.join(rec) + '\n' for rec in records) + + return libdir / record_file From 31906fe64fc0250349eb25940100c0a63a0f6558 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 27 Mar 2023 15:59:34 -0700 Subject: [PATCH 22/67] Bump colcon-core dependency version --- colcon_python_project/task/python/project/build.py | 9 ++++----- setup.cfg | 2 +- stdeb.cfg | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/colcon_python_project/task/python/project/build.py b/colcon_python_project/task/python/project/build.py index ae50a23..2873ccd 100644 --- a/colcon_python_project/task/python/project/build.py +++ b/colcon_python_project/task/python/project/build.py @@ -103,11 +103,10 @@ async def build(self, *, additional_hooks=None): # noqa: D102 scripts = create_environment_scripts( pkg, args, default_hooks=hooks, additional_hooks=additional_hooks) - if scripts: - records += [ - (Path(os.path.relpath(script, libdir)).as_posix(), '', '') - for script in scripts - ] + records += [ + (Path(os.path.relpath(script, libdir)).as_posix(), '', '') + for script in scripts + ] with record_file.open('a') as f: f.writelines(','.join(rec) + '\n' for rec in records) diff --git a/setup.cfg b/setup.cfg index f658b86..91aacaf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,7 @@ keywords = colcon [options] install_requires = - colcon-core + colcon-core>=0.12.0 distlib tomli>=1.0.0;python_version < "3.11" packages = find: diff --git a/stdeb.cfg b/stdeb.cfg index 551e2ca..fbb32c6 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,5 +1,5 @@ [colcon-python-project] No-Python2: -Depends3: python3-colcon-core, python3-distlib, python3-tomli (>= 1.0.0) +Depends3: python3-colcon-core (>= 0.12.0), python3-distlib, python3-tomli (>= 1.0.0) Suite: jammy bullseye X-Python3-Version: >= 3.6 From a84582965232f86dde47e83c013828ea94429943 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 29 Mar 2023 14:40:52 -0700 Subject: [PATCH 23/67] Cache metadata using getter This approach is already used by the existing Python metadata caching to reduce the descriptor size. --- colcon_python_project/metadata.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/colcon_python_project/metadata.py b/colcon_python_project/metadata.py index 26f4a9f..fb9d347 100644 --- a/colcon_python_project/metadata.py +++ b/colcon_python_project/metadata.py @@ -33,8 +33,12 @@ async def load_and_cache_metadata(desc): :param desc: The package descriptor """ - metadata = desc.metadata.get('python_project_metadata') - if metadata is None: + get_metadata = desc.metadata.get('get_python_project_metadata') + if get_metadata is None: metadata = await load_metadata(desc) - desc.metadata['python_project_metadata'] = metadata - return metadata + + def get_metadata(): + return metadata + + desc.metadata['get_python_project_metadata'] = get_metadata + return get_metadata() From 31fed82109a109f3c61a448e655cc9d2b01ff3bb Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 29 Mar 2023 14:42:06 -0700 Subject: [PATCH 24/67] Return dist-info directory path from wheel installation We can easily determine the RECORD path from the dist-info directory, so returning the directory specifically is more straightforward. --- colcon_python_project/task/python/project/build.py | 6 +++--- colcon_python_project/wheel.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/colcon_python_project/task/python/project/build.py b/colcon_python_project/task/python/project/build.py index 2873ccd..ab0fff3 100644 --- a/colcon_python_project/task/python/project/build.py +++ b/colcon_python_project/task/python/project/build.py @@ -88,11 +88,11 @@ async def build(self, *, additional_hooks=None): # noqa: D102 return e.returncode wheel_path = wheel_directory / wheel_name - record_file = install_wheel( + dist_info_dir = install_wheel( wheel_path, args.install_base, script_dir_override=script_dir_override) - libdir = record_file.parent.parent + libdir = dist_info_dir.parent records = [] hooks = create_environment_hooks(args.install_base, pkg.name) @@ -108,7 +108,7 @@ async def build(self, *, additional_hooks=None): # noqa: D102 for script in scripts ] - with record_file.open('a') as f: + with (dist_info_dir / 'RECORD').open('a') as f: f.writelines(','.join(rec) + '\n' for rec in records) def _stdout_callback(self, line): diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index 5da3aff..06e31f7 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -49,7 +49,7 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): :param install_base: Path to the base directory to install under. :param script_dir_override: Override the default script install directory - :returns: Path to the installed RECORD file + :returns: Path to the installed distribution info directory """ wheel_name = wheel_path.name.split('-') if len(wheel_name) not in (5, 6): @@ -133,4 +133,4 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): with (libdir / record_file).open('w') as f: f.writelines(','.join(rec) + '\n' for rec in records) - return libdir / record_file + return libdir / dist_info_dir From 50b6bea88a782da96905c14e935233a8eb013ad7 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 29 Mar 2023 14:43:30 -0700 Subject: [PATCH 25/67] Implement hook listing functionality --- colcon_python_project/_list_hooks.py | 19 +++++++++++++++++++ colcon_python_project/hook_caller.py | 22 ++++++++++++++++++++++ test/test_hook_caller_setuptools.py | 11 +++++++++++ 3 files changed, 52 insertions(+) create mode 100644 colcon_python_project/_list_hooks.py diff --git a/colcon_python_project/_list_hooks.py b/colcon_python_project/_list_hooks.py new file mode 100644 index 0000000..c5551e8 --- /dev/null +++ b/colcon_python_project/_list_hooks.py @@ -0,0 +1,19 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from importlib import import_module +import sys + + +if __name__ == '__main__': + backend_name = sys.argv[1] + if ':' in backend_name: + backend_module_name, backend_object_name = backend_name.split(':', 2) + backend_module = import_module(backend_module_name) + backend = getattr(backend_module, backend_object_name) + else: + backend = import_module(backend_name) + + for attr in dir(backend): + if callable(getattr(backend, attr)): + print(attr) diff --git a/colcon_python_project/hook_caller.py b/colcon_python_project/hook_caller.py index 2b938df..f78f483 100644 --- a/colcon_python_project/hook_caller.py +++ b/colcon_python_project/hook_caller.py @@ -66,6 +66,28 @@ def backend_name(self): """Get the name of the backend to call hooks on.""" return self._backend_name + async def list_hooks(self): + """ + Call into the backend to list implemented hooks. + + This function lists all callable methods on the backend, which may + include more than just the hook names. + + :returns: List of hook names. + """ + args = [ + sys.executable, '-m', 'colcon_python_project._list_hooks', + self._backend_name] + process = await run( + args, None, self._stderr_callback, + cwd=self._project_path, env=self._env, + capture_output=True) + process.check_returncode() + hook_names = [ + line.strip().decode() for line in process.stdout.splitlines()] + return [ + hook for hook in hook_names if hook and not hook.startswith('_')] + async def call_hook(self, hook_name, **kwargs): """ Call the given hook with given arguments. diff --git a/test/test_hook_caller_setuptools.py b/test/test_hook_caller_setuptools.py index b6bb34b..6a71b5b 100644 --- a/test/test_hook_caller_setuptools.py +++ b/test/test_hook_caller_setuptools.py @@ -117,3 +117,14 @@ async def test_prepare_metadata_for_build_editable(mock_project, tmp_path): metadata_directory=str(tmp_path)) assert isinstance(dist_info, str) assert (tmp_path / dist_info).is_dir() + + +@pytest.mark.asyncio +async def test_list_hooks(mock_project): + hook_caller = AsyncHookCaller( + _BACKEND_NAME, project_path=mock_project, + stderr_callback=sys.stderr.buffer.write) + hook_names = await hook_caller.list_hooks() + assert isinstance(hook_names, list) + assert 'build_sdist' in hook_names + assert 'build_wheel' in hook_names From 59668e133c583fc6a015ca2a36bca33fc977fc6d Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 29 Mar 2023 16:47:57 -0700 Subject: [PATCH 26/67] Implement PEP 660 editable installs --- README.rst | 2 +- .../task/python/project/build.py | 25 ++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index c072638..a41ad3c 100644 --- a/README.rst +++ b/README.rst @@ -9,9 +9,9 @@ Extensions for `colcon-core `_ to work wi TODO ---- * Graceful and informational error handling -* Finish PEP 660 (symlink) installs Idiosyncrasies -------------- * For setuptools-based packages, setuptools (< 64.0.0) will leave build artifacts in the source directory. +* For setuptools-based packages, symlink installs always print warnings to stderr with no good way to suppress them. * For poetry-based packages, dependencies expressed in groups (including 'test') are not discovered (use 'test' extra as a workaround). diff --git a/colcon_python_project/task/python/project/build.py b/colcon_python_project/task/python/project/build.py index ab0fff3..8ec83c5 100644 --- a/colcon_python_project/task/python/project/build.py +++ b/colcon_python_project/task/python/project/build.py @@ -80,10 +80,18 @@ async def build(self, *, additional_hooks=None): # noqa: D102 wheel_directory = Path(args.build_base) / 'wheel' wheel_directory.mkdir(parents=True, exist_ok=True) try: + build_hook_name = 'build_wheel' if args.symlink_install: - logger.warn(f'Symlink install is not supported by {__name__}') - wheel_name = await hook_caller.build_wheel( - wheel_directory=wheel_directory) + hook_names = await hook_caller.list_hooks() + if 'build_editable' in hook_names: + build_hook_name = 'build_editable' + else: + logger.info( + f"Backend '{hook_caller.backend_name}' does not " + 'support --symlink-install - falling back to regular ' + "build for package '{pkg.name}'") + wheel_name = await hook_caller.call_hook( + build_hook_name, wheel_directory=wheel_directory) except CalledProcessError as e: return e.returncode @@ -108,6 +116,17 @@ async def build(self, *, additional_hooks=None): # noqa: D102 for script in scripts ] + if build_hook_name == 'build_editable': + # PEP 610 + direct_url_json = dist_info_dir / 'direct_url.json' + with direct_url_json.open('w') as f: + f.write( + f'{{"url":"{pkg.path.absolute().as_uri()}",' + '"dir_info":{"editable":true}}\n') + records.append(( + Path(os.path.relpath(direct_url_json, libdir)).as_posix(), + '', '')) + with (dist_info_dir / 'RECORD').open('a') as f: f.writelines(','.join(rec) + '\n' for rec in records) From 5a1d4a49ca0777f9df6c8678278b5b8517d4d9d2 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 30 Mar 2023 14:45:45 -0700 Subject: [PATCH 27/67] Add usage instructions --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index a41ad3c..1771c4e 100644 --- a/README.rst +++ b/README.rst @@ -15,3 +15,9 @@ Idiosyncrasies * For setuptools-based packages, setuptools (< 64.0.0) will leave build artifacts in the source directory. * For setuptools-based packages, symlink installs always print warnings to stderr with no good way to suppress them. * For poetry-based packages, dependencies expressed in groups (including 'test') are not discovered (use 'test' extra as a workaround). + +Using This Prototype +-------------------- +* To build PEP 517 projects which don't have a ``setup.py`` file, simply install this package as any other colcon extension package. +* To build legacy ``setup.py`` projects using this package instead of the existing python extensions in ``colcon-core`` and ``colcon-python-setup-py``, use the ``COLCON_EXTENSION_BLOCKLIST`` to block them and this package will discover them as a fallback (see `pep517_setuptools_fallback documentation `_). +* To build ``ament_python`` projects using the extensions in this package, use the `colcon-python-project branch `_ of ``colcon-ros``. From bb175cd14704f0fd638b15e8538a8f6227afca20 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 19 May 2023 10:18:11 -0700 Subject: [PATCH 28/67] Invoke hook subprocess as script instead of module Module-based invocation works well when the subprocess interpreter has the same Python PATH and modules as the main process, but this isn't always the case (i.e. pytest). It's safer to import the module we want to call in the main process and directly invoke that script in the subprocess. One behavioral difference between the module and script is that scripts resolve import starting with the script's directory. To avoid collisions, this change moves the scripts into a new subdirectory. --- .../{hook_caller.py => hook_caller/__init__.py} | 6 ++++-- colcon_python_project/{ => hook_caller}/_call_hook.py | 0 colcon_python_project/{ => hook_caller}/_list_hooks.py | 0 3 files changed, 4 insertions(+), 2 deletions(-) rename colcon_python_project/{hook_caller.py => hook_caller/__init__.py} (96%) rename colcon_python_project/{ => hook_caller}/_call_hook.py (100%) rename colcon_python_project/{ => hook_caller}/_list_hooks.py (100%) diff --git a/colcon_python_project/hook_caller.py b/colcon_python_project/hook_caller/__init__.py similarity index 96% rename from colcon_python_project/hook_caller.py rename to colcon_python_project/hook_caller/__init__.py index f78f483..6524c05 100644 --- a/colcon_python_project/hook_caller.py +++ b/colcon_python_project/hook_caller/__init__.py @@ -8,6 +8,8 @@ import sys from colcon_core.subprocess import run +from colcon_python_project.hook_caller import _call_hook +from colcon_python_project.hook_caller import _list_hooks from colcon_python_project.spec import load_and_cache_spec @@ -76,7 +78,7 @@ async def list_hooks(self): :returns: List of hook names. """ args = [ - sys.executable, '-m', 'colcon_python_project._list_hooks', + sys.executable, _list_hooks.__file__, self._backend_name] process = await run( args, None, self._stderr_callback, @@ -96,7 +98,7 @@ async def call_hook(self, hook_name, **kwargs): """ with _SubprocessTransport() as transport: args = [ - sys.executable, '-m', 'colcon_python_project._call_hook', + sys.executable, _call_hook.__file__, self._backend_name, hook_name, str(transport.pass_in), str(transport.pass_out)] with os.fdopen(os.dup(transport.parent_out), 'wb') as f: diff --git a/colcon_python_project/_call_hook.py b/colcon_python_project/hook_caller/_call_hook.py similarity index 100% rename from colcon_python_project/_call_hook.py rename to colcon_python_project/hook_caller/_call_hook.py diff --git a/colcon_python_project/_list_hooks.py b/colcon_python_project/hook_caller/_list_hooks.py similarity index 100% rename from colcon_python_project/_list_hooks.py rename to colcon_python_project/hook_caller/_list_hooks.py From 767bf43b7f8874f060bb6e8818fae01858613da3 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 5 Jun 2023 15:16:28 -0700 Subject: [PATCH 29/67] Add more tests, make the prototype easier to use --- .gitignore | 1 + README.rst | 8 +- .../argument_parser/.python_project.py.swp | Bin 0 -> 12288 bytes .../argument_parser/__init__.py | 0 .../argument_parser/python_project.py | 36 +++++ .../ament_python_project.py | 29 ++++ .../package_identification/pep517.py | 3 + .../task/ament_python/__init__.py | 0 .../task/ament_python/project/__init__.py | 0 .../task/ament_python/project/build.py | 108 +++++++++++++++ setup.cfg | 22 +++ test/__init__.py | 0 test/backend_fixtures/__init__.py | 11 ++ test/backend_fixtures/flit.py | 23 ++++ test/backend_fixtures/legacy.py | 27 ++++ test/backend_fixtures/poetry.py | 21 +++ test/backend_fixtures/setuptools.py | 20 +++ test/conftest.py | 82 +++++++++++ test/spell_check.words | 16 ++- test/test_hook_caller_setuptools.py | 130 ------------------ test/test_hooks.py | 101 ++++++++++++++ test/test_pipeline.py | 87 ++++++++++++ 22 files changed, 588 insertions(+), 137 deletions(-) create mode 100644 colcon_python_project/argument_parser/.python_project.py.swp create mode 100644 colcon_python_project/argument_parser/__init__.py create mode 100644 colcon_python_project/argument_parser/python_project.py create mode 100644 colcon_python_project/package_augmentation/ament_python_project.py create mode 100644 colcon_python_project/task/ament_python/__init__.py create mode 100644 colcon_python_project/task/ament_python/project/__init__.py create mode 100644 colcon_python_project/task/ament_python/project/build.py create mode 100644 test/__init__.py create mode 100644 test/backend_fixtures/__init__.py create mode 100644 test/backend_fixtures/flit.py create mode 100644 test/backend_fixtures/legacy.py create mode 100644 test/backend_fixtures/poetry.py create mode 100644 test/backend_fixtures/setuptools.py create mode 100644 test/conftest.py delete mode 100644 test/test_hook_caller_setuptools.py create mode 100644 test/test_hooks.py create mode 100644 test/test_pipeline.py diff --git a/.gitignore b/.gitignore index c18dd8d..4eff406 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +/.coverage __pycache__/ diff --git a/README.rst b/README.rst index 1771c4e..f563e26 100644 --- a/README.rst +++ b/README.rst @@ -9,15 +9,11 @@ Extensions for `colcon-core `_ to work wi TODO ---- * Graceful and informational error handling +* Uninstall previously installed wheels before installing the newly built ones +* More tests Idiosyncrasies -------------- * For setuptools-based packages, setuptools (< 64.0.0) will leave build artifacts in the source directory. * For setuptools-based packages, symlink installs always print warnings to stderr with no good way to suppress them. * For poetry-based packages, dependencies expressed in groups (including 'test') are not discovered (use 'test' extra as a workaround). - -Using This Prototype --------------------- -* To build PEP 517 projects which don't have a ``setup.py`` file, simply install this package as any other colcon extension package. -* To build legacy ``setup.py`` projects using this package instead of the existing python extensions in ``colcon-core`` and ``colcon-python-setup-py``, use the ``COLCON_EXTENSION_BLOCKLIST`` to block them and this package will discover them as a fallback (see `pep517_setuptools_fallback documentation `_). -* To build ``ament_python`` projects using the extensions in this package, use the `colcon-python-project branch `_ of ``colcon-ros``. diff --git a/colcon_python_project/argument_parser/.python_project.py.swp b/colcon_python_project/argument_parser/.python_project.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..cf38b803325987188ebdc3189756b7cfb79f72cc GIT binary patch literal 12288 zcmeI2&yO256vy2n4*V#f@(1v`d&mT3XS3;{uu_rIZi=GrCQ)`9AuYw4opBPk6MJeq zyA2ZMAD|}=aO42x2!8?``5zGf0&wNTn~al?EZbJSwS6OfW%<4F`1#{!OHtlYwb$H( z55p?M=N-oW+50@HKMdK|7a5DSb2ggfp2t4^y^T~TeXPSoq}fo0-Wn?5{OAYXv>EF( z){39lO>1J`+sGzvsMSVh^c@kq**k@m&#aLdj7LH_o<+tAv$^aJvnL1p*p-lz2)v*K z#;(8d!G&9wYg^N!zP@$|-o1G11zjvHiwF<_B0vO)01+SpM1Tkof&U}mvUBW9be#9j zl5fwgw7yFX5g-CYfCvx)B0vO)01+SpM1Tko0V41m5{MGUeteCw!`Cr*{QrOU`~SDM z8T%IX4eBB4E7V7*_fUVl#n>a%52)`^_faj>P1Gx>^Qhl1F!n3z5$X{2G3o~DRn*^a zGWHAV0qP6X71UpEF!m?vJ5&eNMlsaS7}rlIgSv;B#`zp_rjUsM5g-CYfCvx)BJdIr z0RI@_#zsL=vB}uXtZgXNu{2s8UfXS7zsY;`Zm6P>xD8hUomu3DR%GEFEmd%o>iDiL zFGlBIDU4y1WkMxE$yb%e+6WvR-;D->%LI!>WnadTlUiYYXkKH=Ek=~_Y*H%Eqmg}h zmJfi^4y5v-Y`+}wJu9}-Q_w2*{*1&3ZI((GERths{}u@q4ulJqld1wOtwecyzTrjk z;B<$#r?y1&ArY87krRBS%s8+j?N{LaicjM5Ho!VA=3cZ7J6qM-^0t>g+X}1yVT73q zbzisKnU*c@6!U1)xbK9rn3)|+au+r}IcT)@o9!0ww41F1-fML8US%G%#Wc2kDJ(x0 zCLb3pT;oibpd4~8m2{j31)F?Xb5p*ImDsNov9GPI?MNFPr2@n(CJQG+84m$Vwfm`z z9h^v)_ww59m1^w}$1@N2sg}y(G_n>t)AQIVO1{q+7~oaeH+tk}=EeC=#{-E6d19Ry zfgEMpI5=0.12.0 + colcon-ros distlib tomli>=1.0.0;python_version < "3.11" packages = find: @@ -46,23 +47,34 @@ test = pylint pytest pytest-asyncio + pytest-benchmark pytest-cov scspell3k>=2.2 setuptools>=40.8.0 +[options.packages.find] +exclude = + test + test.* + [tool:pytest] +addopts = --benchmark-disable filterwarnings = error # Suppress deprecation warnings in other packages ignore:lib2to3 package is deprecated::scspell ignore:SelectableGroups dict interface is deprecated::flake8 ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated::pyreadline + ignore:ResourceWarning. unclosed::_pytest junit_suite_name = colcon-python-project [options.entry_points] +colcon_core.argument_parser = + python_project = colcon_python_project.argument_parser.python_project:PythonProjectArgumentParserDecorator colcon_core.extension_point = colcon_python_project.hook_caller_decorator = colcon_python_project.hook_caller_decorator:HookCallerDecoratorExtensionPoint colcon_core.package_augmentation = + ament_python_project = colcon_python_project.package_augmentation.ament_python_project:RosAmentPythonProjectPackageAugmentation pep517 = colcon_python_project.package_augmentation.pep517:PEP517PackageAugmentation pep621 = colcon_python_project.package_augmentation.pep621:PEP621PackageAugmentation colcon_core.package_identification = @@ -71,8 +83,10 @@ colcon_core.package_identification = pep621 = colcon_python_project.package_identification.pep621:PEP621PackageIdentification colcon_core.task.build = python.project = colcon_python_project.task.python.project.build:PythonProjectBuildTask + ros.ament_python.project = colcon_python_project.task.ament_python.project.build:AmentPythonProjectBuildTask colcon_core.task.test = python.project = colcon_python_project.task.python.project.test:PythonProjectTestTask + ros.ament_python.project = colcon_ros.task.ament_python.test:AmentPythonTestTask colcon_python_project.hook_caller_decorator = setuptools = colcon_python_project.hook_caller_decorator.setuptools:SetuptoolsHookCallerDecoratorExtension @@ -81,3 +95,11 @@ import-order-style = google [coverage:run] source = colcon_python_project +omit = + setup.py + test/* + +[coverage:report] +omit = + setup.py + test/* diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/backend_fixtures/__init__.py b/test/backend_fixtures/__init__.py new file mode 100644 index 0000000..1b2f3d7 --- /dev/null +++ b/test/backend_fixtures/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from .flit import mock_backend_project as mock_flit_project # noqa: F401 +from .legacy import mock_backend_project as mock_legacy_project # noqa: F401 +from .poetry import mock_backend_project as mock_poetry_project # noqa: F401 +from .setuptools \ + import mock_backend_project as mock_setuptools_project # noqa: F401 + + +MOCK_BACKENDS = ('flit', 'legacy', 'poetry', 'setuptools') diff --git a/test/backend_fixtures/flit.py b/test/backend_fixtures/flit.py new file mode 100644 index 0000000..1ca9315 --- /dev/null +++ b/test/backend_fixtures/flit.py @@ -0,0 +1,23 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import pytest + + +@pytest.fixture +def mock_backend_project(mock_desc): + with (mock_desc.path / 'pyproject.toml').open('w') as f: + f.write('\n'.join([ + '[build-system]', + 'requires = ["flit_core >=3.2,<4"]', + 'build-backend = "flit_core.buildapi"', + '', + '[project]', + f'name = "{mock_desc.name}"', + 'version = "0.0.0"', + 'description = "A test project"', + '', + '[tool.flit.module]', + 'name = "test_project"', + ])) + return mock_desc diff --git a/test/backend_fixtures/legacy.py b/test/backend_fixtures/legacy.py new file mode 100644 index 0000000..27a9cac --- /dev/null +++ b/test/backend_fixtures/legacy.py @@ -0,0 +1,27 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import pytest + + +@pytest.fixture +def mock_backend_project(mock_desc): + with (mock_desc.path / 'setup.cfg').open('w') as f: + f.write('\n'.join([ + '[metadata]', + f'name = {mock_desc.name}', + '[options]', + 'packages = find:', + ])) + with (mock_desc.path / 'setup.py').open('w') as f: + f.write('\n'.join([ + 'from setuptools import setup', + 'setup()', + ])) + with (mock_desc.path / 'pyproject.toml').open('w') as f: + f.write('\n'.join([ + '[build-system]', + 'requires = ["setuptools>=40.8.0"]', + 'build-backend = "setuptools.build_meta:__legacy__"', + ])) + return mock_desc diff --git a/test/backend_fixtures/poetry.py b/test/backend_fixtures/poetry.py new file mode 100644 index 0000000..feadd4b --- /dev/null +++ b/test/backend_fixtures/poetry.py @@ -0,0 +1,21 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import pytest + + +@pytest.fixture +def mock_backend_project(mock_desc): + with (mock_desc.path / 'pyproject.toml').open('w') as f: + f.write('\n'.join([ + '[build-system]', + 'requires = ["poetry-core"]', + 'build-backend = "poetry.core.masonry.api"', + '', + '[tool.poetry]', + f'name = "{mock_desc.name}"', + 'version = "0.0.0"', + 'description = "A test project"', + 'authors = ["Author "]', + ])) + return mock_desc diff --git a/test/backend_fixtures/setuptools.py b/test/backend_fixtures/setuptools.py new file mode 100644 index 0000000..65e3ae2 --- /dev/null +++ b/test/backend_fixtures/setuptools.py @@ -0,0 +1,20 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import pytest + + +@pytest.fixture +def mock_backend_project(mock_desc): + with (mock_desc.path / 'pyproject.toml').open('w') as f: + f.write('\n'.join([ + '[build-system]', + 'requires = ["setuptools>=61.0.0"]', + 'build-backend = "setuptools.build_meta"', + '', + '[project]', + f'name = "{mock_desc.name}"', + 'version = "0.0.0"', + 'description = "A test project"', + ])) + return mock_desc diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..d544010 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,82 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import logging +import os +import unittest.mock +import warnings + +from colcon_core.package_descriptor import PackageDescriptor +from colcon_python_project.hook_caller_decorator.setuptools \ + import SetuptoolsHookCallerDecoratorExtension +import pytest + +from .backend_fixtures import * # noqa: F401, F403 +from .backend_fixtures import MOCK_BACKENDS + + +@pytest.fixture(autouse=True) +def better_benchmarking(request): + if 'benchmark' not in request.fixturenames: + yield + return + benchmark = request.getfixturevalue('benchmark') + if not benchmark.enabled: + yield + return + if 'COV_CORE_DATAFILE' in os.environ: + warnings.warn( + "Can't run benchmarks with coverage enabled. " + 'Re-run with --no-cov.') + benchmark.disabled = True + yield + return + logging.disable(level=logging.ERROR) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + yield + logging.disable(level=logging.NOTSET) + + +@pytest.fixture(autouse=True, scope='session') +def static_extensions(): + def override_extensions(group_name, *args, **kwargs): + if group_name == 'colcon_python_project.hook_caller_decorator': + return { + 'setuptools': SetuptoolsHookCallerDecoratorExtension(), + } + return unittest.mock.DEFAULT + + with unittest.mock.patch( + 'colcon_python_project.hook_caller_decorator.instantiate_extensions', + side_effect=override_extensions, + ) as p: + yield p + + +@pytest.fixture +def bench(benchmark, event_loop): + def res(target, *args, **kwargs): + def dut(): + return event_loop.run_until_complete(target(*args, **kwargs)) + return benchmark(dut) + return res + + +@pytest.fixture +def mock_desc(tmp_path): + src = tmp_path / 'src' + src.mkdir() + with (src / 'test_project.py').open('w'): + pass + with (src / 'README').open('w'): + pass + desc = PackageDescriptor(src) + desc.name = 'test-project' + desc.type = 'python.project' + return desc + + +@pytest.fixture(params=MOCK_BACKENDS) +def mock_project(request): + return request.getfixturevalue(f'mock_{request.param}_project') diff --git a/test/spell_check.words b/test/spell_check.words index 922e194..5fa35a4 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -1,36 +1,48 @@ +ament apache asyncio +augmentor +autouse backend backends blocklist +buildapi colcon configparser +conftest contextlib copyfileobj cottsay +datafile decoree +deepcopy deps -distinfo distlib fdopen fdst +fixturenames fsrc functools +getfixturevalue hashlib importlib iterdir libdir +mktemp namelist noqa +notset osfhandle partialmethod pathlib platlib plugin +prepend purelib pyproject pytest pythonpath +pythonwarnings relpath returncode rstrip @@ -38,6 +50,7 @@ rtype scspell sdist setuptools +simplefilter subpath symlink tempfile @@ -47,5 +60,6 @@ toml tomli tomllib traceback +unittest urlsafe zipfile diff --git a/test/test_hook_caller_setuptools.py b/test/test_hook_caller_setuptools.py deleted file mode 100644 index 6a71b5b..0000000 --- a/test/test_hook_caller_setuptools.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2022 Open Source Robotics Foundation, Inc. -# Licensed under the Apache License, Version 2.0 - -import sys - -from colcon_python_project.hook_caller import AsyncHookCaller -import pytest - -_BACKEND_NAME = 'setuptools.build_meta:__legacy__' - - -@pytest.fixture -def mock_project(tmp_path): - with (tmp_path / 'test_project.py').open('w'): - pass - with (tmp_path / 'README').open('w'): - pass - with (tmp_path / 'setup.cfg').open('w') as f: - f.write('\n'.join([ - '[metadata]', - 'name = test-project', - '[options]', - 'packages = find:', - ])) - with (tmp_path / 'setup.py').open('w') as f: - f.write('\n'.join([ - 'from setuptools import setup', - 'setup()', - ])) - with (tmp_path / 'pyproject.toml').open('w') as f: - f.write('\n'.join([ - '[build-system]', - 'requires = ["setuptools>=40.8.0"]', - f'build-backend = "{_BACKEND_NAME}"', - ])) - yield tmp_path - - -@pytest.mark.asyncio -async def test_build_wheel(mock_project, tmp_path): - hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project, - stderr_callback=sys.stderr.buffer.write) - wheel = await hook_caller.build_wheel(wheel_directory=str(tmp_path)) - assert isinstance(wheel, str) - assert (tmp_path / wheel).is_file() - - -@pytest.mark.asyncio -async def test_build_sdist(mock_project, tmp_path): - hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project, - stderr_callback=sys.stderr.buffer.write) - sdist = await hook_caller.build_sdist(sdist_directory=str(tmp_path)) - assert isinstance(sdist, str) - assert (tmp_path / sdist).is_file() - - -@pytest.mark.asyncio -async def test_get_requires_for_build_wheel(mock_project): - hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project, - stderr_callback=sys.stderr.buffer.write) - requires = await hook_caller.get_requires_for_build_wheel() - assert isinstance(requires, list) - - -@pytest.mark.asyncio -async def test_prepare_metadata_for_build_wheel(mock_project, tmp_path): - hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project, - stderr_callback=sys.stderr.buffer.write) - distinfo = await hook_caller.prepare_metadata_for_build_wheel( - metadata_directory=str(tmp_path)) - assert isinstance(distinfo, str) - assert (tmp_path / distinfo).is_dir() - - -@pytest.mark.asyncio -async def test_get_requires_for_build_sdist(mock_project): - hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project, - stderr_callback=sys.stderr.buffer.write) - requires = await hook_caller.get_requires_for_build_sdist() - assert isinstance(requires, list) - - -@pytest.mark.asyncio -@pytest.mark.skip(reason='Insufficient setuptools version') -async def test_build_editable(mock_project, tmp_path): - hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project, - stderr_callback=sys.stderr.buffer.write) - wheel = await hook_caller.build_editable( - wheel_directory=str(tmp_path)) - assert isinstance(wheel, str) - assert (tmp_path / wheel).is_file() - - -@pytest.mark.asyncio -@pytest.mark.skip(reason='Insufficient setuptools version') -async def test_get_requires_for_build_editable(mock_project): - hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project, - stderr_callback=sys.stderr.buffer.write) - requires = await hook_caller.get_requires_for_build_editable() - assert isinstance(requires, list) - - -@pytest.mark.asyncio -@pytest.mark.skip(reason='Insufficient setuptools version') -async def test_prepare_metadata_for_build_editable(mock_project, tmp_path): - hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project, - stderr_callback=sys.stderr.buffer.write) - dist_info = await hook_caller.prepare_metadata_for_build_editable( - metadata_directory=str(tmp_path)) - assert isinstance(dist_info, str) - assert (tmp_path / dist_info).is_dir() - - -@pytest.mark.asyncio -async def test_list_hooks(mock_project): - hook_caller = AsyncHookCaller( - _BACKEND_NAME, project_path=mock_project, - stderr_callback=sys.stderr.buffer.write) - hook_names = await hook_caller.list_hooks() - assert isinstance(hook_names, list) - assert 'build_sdist' in hook_names - assert 'build_wheel' in hook_names diff --git a/test/test_hooks.py b/test/test_hooks.py new file mode 100644 index 0000000..beb43de --- /dev/null +++ b/test/test_hooks.py @@ -0,0 +1,101 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import os +import sys + +from colcon_python_project.hook_caller_decorator \ + import get_decorated_hook_caller +import pytest + + +@pytest.fixture +def mock_hook_caller(mock_project): + env = { + **os.environ, + 'PYTHONWARNINGS': 'ignore', + } + return get_decorated_hook_caller( + mock_project, + env=env, stderr_callback=sys.stderr.buffer.write) + + +@pytest.mark.benchmark(group='hooks.build_wheel') +def test_build_wheel(mock_hook_caller, tmp_path_factory, bench): + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / await mock_hook_caller.build_wheel( + wheel_directory=str(out)) + wheel = bench(dut) + assert wheel.is_file() + + +@pytest.mark.benchmark(group='hooks.build_sdist') +def test_build_sdist(mock_hook_caller, tmp_path_factory, bench): + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / await mock_hook_caller.build_sdist( + sdist_directory=str(out)) + sdist = bench(dut) + assert sdist.is_file() + + +@pytest.mark.benchmark(group='hooks.get_requires_for_build_wheel') +def test_get_requires_for_build_wheel(mock_hook_caller, bench): + requires = bench(mock_hook_caller.get_requires_for_build_wheel) + assert isinstance(requires, list) + + +@pytest.mark.benchmark(group='hooks.prepare_metadata_for_build_wheel') +def test_prepare_metadata_for_build_wheel( + mock_hook_caller, tmp_path_factory, bench +): + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / await mock_hook_caller.prepare_metadata_for_build_wheel( + metadata_directory=str(out)) + dist_info = bench(dut) + assert dist_info.is_dir() + + +@pytest.mark.benchmark(group='hooks.get_requires_for_build_sdist') +def test_get_requires_for_build_sdist(mock_hook_caller, bench): + requires = bench(mock_hook_caller.get_requires_for_build_sdist) + assert isinstance(requires, list) + + +@pytest.mark.benchmark(group='hooks.build_editable') +def test_build_editable(mock_hook_caller, tmp_path_factory, bench): + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / await mock_hook_caller.build_editable( + wheel_directory=str(out)) + wheel = bench(dut) + assert wheel.is_file() + + +@pytest.mark.benchmark(group='hooks.get_requires_for_build_editable') +def test_get_requires_for_build_editable(mock_hook_caller, bench): + requires = bench(mock_hook_caller.get_requires_for_build_editable) + assert isinstance(requires, list) + + +@pytest.mark.benchmark(group='hooks.prepare_metadata_for_build_editable') +def test_prepare_metadata_for_build_editable( + mock_hook_caller, tmp_path_factory, bench +): + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / \ + await mock_hook_caller.prepare_metadata_for_build_editable( + metadata_directory=str(out)) + dist_info = bench(dut) + assert dist_info.is_dir() + + +@pytest.mark.benchmark(group='hooks.list_hooks') +def test_list_hooks(mock_hook_caller, bench): + hook_names = bench(mock_hook_caller.list_hooks) + assert isinstance(hook_names, list) + assert 'build_sdist' in hook_names + assert 'build_wheel' in hook_names diff --git a/test/test_pipeline.py b/test/test_pipeline.py new file mode 100644 index 0000000..b28f159 --- /dev/null +++ b/test/test_pipeline.py @@ -0,0 +1,87 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from copy import deepcopy + +from colcon_core.package_augmentation.python \ + import PythonPackageAugmentation +from colcon_core.package_descriptor import PackageDescriptor +from colcon_core.package_identification.python \ + import PythonPackageIdentification +from colcon_python_project.package_augmentation.pep517 \ + import PEP517PackageAugmentation +from colcon_python_project.package_augmentation.pep621 \ + import PEP621PackageAugmentation +from colcon_python_project.package_identification.pep517 \ + import PEP517PackageIdentification +from colcon_python_project.package_identification.pep517_setuptools_fallback \ + import PEP517SetuptoolsFallbackPackageIdentification +from colcon_python_project.package_identification.pep621 \ + import PEP621PackageIdentification +from colcon_python_setup_py.package_augmentation.python_setup_py \ + import PythonPackageAugmentation as SetupPyPackageAugmentation +from colcon_python_setup_py.package_identification.python_setup_py \ + import PythonPackageIdentification as SetupPyPackageIdentification +import pytest + +from .conftest import MOCK_BACKENDS + + +PIPELINES = { + **{ + f'pep517.{backend}': ( + backend, + PEP517PackageIdentification(), + PEP517PackageAugmentation()) + for backend in MOCK_BACKENDS + }, + 'pep517.setuptools_fallback': ( + 'legacy', + PEP517SetuptoolsFallbackPackageIdentification(), + PEP517PackageAugmentation()), + 'pep621': ( + 'flit', + PEP621PackageIdentification(), + PEP621PackageAugmentation()), + 'setup_cfg': ( + 'legacy', + PythonPackageIdentification(), + PythonPackageAugmentation()), + 'setup_py': ( + 'legacy', + SetupPyPackageIdentification(), + SetupPyPackageAugmentation()), +} + + +@pytest.mark.benchmark(group='pipeline.package_identification') +@pytest.mark.parametrize( + 'backend,identifier,_', + PIPELINES.values(), + ids=PIPELINES.keys()) +def test_identify(backend, identifier, _, request, benchmark): + mock_project = request.getfixturevalue(f'mock_{backend}_project') + + def dut(): + mock_desc = PackageDescriptor(mock_project.path) + assert identifier.identify(mock_desc) is None + return mock_desc + mock_desc = benchmark(dut) + assert mock_desc.identifies_package() + + +@pytest.mark.benchmark(group='pipeline.package_augmentation') +@pytest.mark.parametrize( + 'backend,identifier,augmentor', + PIPELINES.values(), + ids=PIPELINES.keys()) +def test_augment(backend, identifier, augmentor, request, benchmark): + mock_project = request.getfixturevalue(f'mock_{backend}_project') + mock_project = PackageDescriptor(mock_project.path) + assert identifier.identify(mock_project) is None + assert mock_project.identifies_package() + + @benchmark + def dut(): + mock_desc = deepcopy(mock_project) + assert augmentor.augment_package(mock_desc) is None From 78d3daabdfa95ed0542e165843550f43bfd1a9bd Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 6 Jun 2023 10:48:51 -0700 Subject: [PATCH 30/67] Close event loops to avoid ResourceWarning --- colcon_python_project/package_augmentation/pep517.py | 5 +++++ colcon_python_project/package_identification/pep517.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/colcon_python_project/package_augmentation/pep517.py b/colcon_python_project/package_augmentation/pep517.py index ccf201c..59152b2 100644 --- a/colcon_python_project/package_augmentation/pep517.py +++ b/colcon_python_project/package_augmentation/pep517.py @@ -54,6 +54,8 @@ def augment_package( # noqa: D102 logger.warn( f'An error occurred while reading metadata for {desc.name}:' f" {e.stderr.strip().decode() or '(no output)'}") + loop.stop() + loop.close() return desc.dependencies.setdefault('build', set()) desc.dependencies['build'].update( @@ -67,6 +69,9 @@ def augment_package( # noqa: D102 f'An error occurred while reading metadata for {desc.name}:' f" {e.stderr.strip().decode() or '(no output)'}") return + finally: + loop.stop() + loop.close() desc.dependencies.setdefault('run', set()) desc.dependencies.setdefault('test', set()) for raw_req in metadata.get_all('Requires-Dist', ()): diff --git a/colcon_python_project/package_identification/pep517.py b/colcon_python_project/package_identification/pep517.py index bf74e0d..06d8d8d 100644 --- a/colcon_python_project/package_identification/pep517.py +++ b/colcon_python_project/package_identification/pep517.py @@ -58,9 +58,9 @@ def identify(self, desc): # noqa: D102 f'An error occurred while reading metadata for {desc.path}:' f" {e.stderr.strip().decode() or '(no output)'}") return - # finally: - # loop.stop() - # loop.close() + finally: + loop.stop() + loop.close() name = metadata.get('Name') if not name: return From 4a109bdbb8c60376ea41e9a0b35f6b3546ad18d7 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 6 Jun 2023 09:19:40 -0700 Subject: [PATCH 31/67] Suppress an upstream deprecation --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 989c2a4..191b35f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,7 +65,7 @@ filterwarnings = ignore:lib2to3 package is deprecated::scspell ignore:SelectableGroups dict interface is deprecated::flake8 ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated::pyreadline - ignore:ResourceWarning. unclosed::_pytest + ignore:pkg_resources is deprecated as an API::pkg_resources junit_suite_name = colcon-python-project [options.entry_points] From fc4e67b2dc738eecb2761cb94838276d6f8ab7a7 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 6 Jun 2023 11:35:09 -0700 Subject: [PATCH 32/67] Set the thread's event loop --- colcon_python_project/package_augmentation/pep517.py | 4 ++++ colcon_python_project/package_identification/pep517.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/colcon_python_project/package_augmentation/pep517.py b/colcon_python_project/package_augmentation/pep517.py index 59152b2..3d3feaf 100644 --- a/colcon_python_project/package_augmentation/pep517.py +++ b/colcon_python_project/package_augmentation/pep517.py @@ -1,6 +1,7 @@ # Copyright 2022 Open Source Robotics Foundation, Inc. # Licensed under the Apache License, Version 2.0 +import asyncio import logging from subprocess import CalledProcessError @@ -45,6 +46,7 @@ def augment_package( # noqa: D102 return loop = new_event_loop() + asyncio.set_event_loop(loop) hook_caller = get_decorated_hook_caller(desc) # TODO(cottsay): get_requires_for_build_editable try: @@ -54,6 +56,7 @@ def augment_package( # noqa: D102 logger.warn( f'An error occurred while reading metadata for {desc.name}:' f" {e.stderr.strip().decode() or '(no output)'}") + asyncio.set_event_loop(None) loop.stop() loop.close() return @@ -70,6 +73,7 @@ def augment_package( # noqa: D102 f" {e.stderr.strip().decode() or '(no output)'}") return finally: + asyncio.set_event_loop(None) loop.stop() loop.close() desc.dependencies.setdefault('run', set()) diff --git a/colcon_python_project/package_identification/pep517.py b/colcon_python_project/package_identification/pep517.py index 06d8d8d..0ed7326 100644 --- a/colcon_python_project/package_identification/pep517.py +++ b/colcon_python_project/package_identification/pep517.py @@ -1,6 +1,7 @@ # Copyright 2022 Open Source Robotics Foundation, Inc. # Licensed under the Apache License, Version 2.0 +import asyncio import logging from subprocess import CalledProcessError @@ -50,6 +51,7 @@ def identify(self, desc): # noqa: D102 return loop = new_event_loop() + asyncio.set_event_loop(loop) try: metadata = loop.run_until_complete( load_and_cache_metadata(desc)) @@ -59,6 +61,7 @@ def identify(self, desc): # noqa: D102 f" {e.stderr.strip().decode() or '(no output)'}") return finally: + asyncio.set_event_loop(None) loop.stop() loop.close() name = metadata.get('Name') From 95f266c40479c3adb156a09e23c65d0c3315f2d7 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 6 Jun 2023 13:08:29 -0700 Subject: [PATCH 33/67] Update test requirements --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 191b35f..d83ba46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,7 +43,9 @@ test = flake8-docstrings flake8-import-order flake8-quotes + flit pep8-naming + poetry pylint pytest pytest-asyncio @@ -51,6 +53,7 @@ test = pytest-cov scspell3k>=2.2 setuptools>=40.8.0 + wheel [options.packages.find] exclude = From d469873f62845c042406ca39acbe1b1d30383f9d Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 6 Jun 2023 13:23:11 -0700 Subject: [PATCH 34/67] Python 3.6 compatibility --- .../hook_caller_decorator/setuptools.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/colcon_python_project/hook_caller_decorator/setuptools.py b/colcon_python_project/hook_caller_decorator/setuptools.py index 87d67c4..3a58fca 100644 --- a/colcon_python_project/hook_caller_decorator/setuptools.py +++ b/colcon_python_project/hook_caller_decorator/setuptools.py @@ -72,12 +72,10 @@ class SetuptoolsDecorator(GenericDecorator): async def build_wheel(self, **kwargs): # noqa: D102 config_settings = kwargs.pop('config_settings', {}) - with ( - _ScratchEggBase(config_settings), - _ScratchBuildBase(config_settings), - ): - return await self._decoree.build_wheel( - config_settings=config_settings, **kwargs) + with _ScratchEggBase(config_settings): + with _ScratchBuildBase(config_settings): + return await self._decoree.build_wheel( + config_settings=config_settings, **kwargs) async def get_requires_for_build_wheel(self, **kwargs): # noqa: D102 config_settings = kwargs.pop('config_settings', {}) From 21e7d1dc1bf3edfa89c0b4cf776a37fb9adc0959 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 6 Jun 2023 13:32:27 -0700 Subject: [PATCH 35/67] Let the Python 3.6 tests fail for now --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 709bbb1..c13df91 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,6 +8,7 @@ on: jobs: setup: runs-on: ubuntu-latest + continue-on-error: ${{ matrix.python == '3.6' }} outputs: strategy: ${{steps.load.outputs.strategy}} From d0b03c61d8e31d6cfbcc01c00caf577e17d77c52 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 9 Jun 2023 10:57:14 -0700 Subject: [PATCH 36/67] Deal with older setuptools versions --- test/spell_check.words | 2 ++ test/test_hooks.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/test/spell_check.words b/test/spell_check.words index 5fa35a4..9abba0b 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -26,8 +26,10 @@ functools getfixturevalue hashlib importlib +importorskip iterdir libdir +minversion mktemp namelist noqa diff --git a/test/test_hooks.py b/test/test_hooks.py index beb43de..ca14236 100644 --- a/test/test_hooks.py +++ b/test/test_hooks.py @@ -66,6 +66,9 @@ def test_get_requires_for_build_sdist(mock_hook_caller, bench): @pytest.mark.benchmark(group='hooks.build_editable') def test_build_editable(mock_hook_caller, tmp_path_factory, bench): + if mock_hook_caller.backend_name.startswith('setuptools.'): + pytest.importorskip('setuptools', minversion='64.0.0') + async def dut(): out = tmp_path_factory.mktemp('out') return out / await mock_hook_caller.build_editable( @@ -76,6 +79,9 @@ async def dut(): @pytest.mark.benchmark(group='hooks.get_requires_for_build_editable') def test_get_requires_for_build_editable(mock_hook_caller, bench): + if mock_hook_caller.backend_name.startswith('setuptools.'): + pytest.importorskip('setuptools', minversion='64.0.0') + requires = bench(mock_hook_caller.get_requires_for_build_editable) assert isinstance(requires, list) @@ -84,6 +90,9 @@ def test_get_requires_for_build_editable(mock_hook_caller, bench): def test_prepare_metadata_for_build_editable( mock_hook_caller, tmp_path_factory, bench ): + if mock_hook_caller.backend_name.startswith('setuptools.'): + pytest.importorskip('setuptools', minversion='64.0.0') + async def dut(): out = tmp_path_factory.mktemp('out') return out / \ From f613ef9b1ba07dcd4e6d0822244237e3666f0de8 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 9 Jun 2023 10:37:45 -0700 Subject: [PATCH 37/67] Use colcon's new_event_loop for async benchmarking --- test/conftest.py | 17 +++++++++++++++-- test/spell_check.words | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index d544010..8c51bf3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,12 +1,14 @@ # Copyright 2023 Open Source Robotics Foundation, Inc. # Licensed under the Apache License, Version 2.0 +import asyncio import logging import os import unittest.mock import warnings from colcon_core.package_descriptor import PackageDescriptor +from colcon_core.subprocess import new_event_loop from colcon_python_project.hook_caller_decorator.setuptools \ import SetuptoolsHookCallerDecoratorExtension import pytest @@ -15,6 +17,16 @@ from .backend_fixtures import MOCK_BACKENDS +@pytest.fixture +def colcon_event_loop(): + loop = new_event_loop() + asyncio.set_event_loop(loop) + yield loop + loop.run_until_complete(loop.shutdown_asyncgens()) + asyncio.set_event_loop(None) + loop.close() + + @pytest.fixture(autouse=True) def better_benchmarking(request): if 'benchmark' not in request.fixturenames: @@ -55,10 +67,11 @@ def override_extensions(group_name, *args, **kwargs): @pytest.fixture -def bench(benchmark, event_loop): +def bench(benchmark, colcon_event_loop): def res(target, *args, **kwargs): def dut(): - return event_loop.run_until_complete(target(*args, **kwargs)) + return colcon_event_loop.run_until_complete( + target(*args, **kwargs)) return benchmark(dut) return res diff --git a/test/spell_check.words b/test/spell_check.words index 9abba0b..eacc7bd 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -1,5 +1,6 @@ ament apache +asyncgens asyncio augmentor autouse From 7b9acc9497e7100b733f0c36afad5cac01d3cb47 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 9 Jun 2023 13:12:39 -0700 Subject: [PATCH 38/67] Drop dependency on pytest-asyncio --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d83ba46..d8eee9a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,7 +48,6 @@ test = poetry pylint pytest - pytest-asyncio pytest-benchmark pytest-cov scspell3k>=2.2 From 8d761b7090653728848397f5589ce5d758f9e5fa Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 9 Jun 2023 14:19:00 -0700 Subject: [PATCH 39/67] Revert "Let the Python 3.6 tests fail for now" This reverts commit 21e7d1dc1bf3edfa89c0b4cf776a37fb9adc0959. --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c13df91..709bbb1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,7 +8,6 @@ on: jobs: setup: runs-on: ubuntu-latest - continue-on-error: ${{ matrix.python == '3.6' }} outputs: strategy: ${{steps.load.outputs.strategy}} From b8ed4e1145ef72e705a19c41954e30f9b918134b Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 9 Jun 2023 14:19:46 -0700 Subject: [PATCH 40/67] How did that get there? --- .../argument_parser/.python_project.py.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 colcon_python_project/argument_parser/.python_project.py.swp diff --git a/colcon_python_project/argument_parser/.python_project.py.swp b/colcon_python_project/argument_parser/.python_project.py.swp deleted file mode 100644 index cf38b803325987188ebdc3189756b7cfb79f72cc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2&yO256vy2n4*V#f@(1v`d&mT3XS3;{uu_rIZi=GrCQ)`9AuYw4opBPk6MJeq zyA2ZMAD|}=aO42x2!8?``5zGf0&wNTn~al?EZbJSwS6OfW%<4F`1#{!OHtlYwb$H( z55p?M=N-oW+50@HKMdK|7a5DSb2ggfp2t4^y^T~TeXPSoq}fo0-Wn?5{OAYXv>EF( z){39lO>1J`+sGzvsMSVh^c@kq**k@m&#aLdj7LH_o<+tAv$^aJvnL1p*p-lz2)v*K z#;(8d!G&9wYg^N!zP@$|-o1G11zjvHiwF<_B0vO)01+SpM1Tkof&U}mvUBW9be#9j zl5fwgw7yFX5g-CYfCvx)B0vO)01+SpM1Tko0V41m5{MGUeteCw!`Cr*{QrOU`~SDM z8T%IX4eBB4E7V7*_fUVl#n>a%52)`^_faj>P1Gx>^Qhl1F!n3z5$X{2G3o~DRn*^a zGWHAV0qP6X71UpEF!m?vJ5&eNMlsaS7}rlIgSv;B#`zp_rjUsM5g-CYfCvx)BJdIr z0RI@_#zsL=vB}uXtZgXNu{2s8UfXS7zsY;`Zm6P>xD8hUomu3DR%GEFEmd%o>iDiL zFGlBIDU4y1WkMxE$yb%e+6WvR-;D->%LI!>WnadTlUiYYXkKH=Ek=~_Y*H%Eqmg}h zmJfi^4y5v-Y`+}wJu9}-Q_w2*{*1&3ZI((GERths{}u@q4ulJqld1wOtwecyzTrjk z;B<$#r?y1&ArY87krRBS%s8+j?N{LaicjM5Ho!VA=3cZ7J6qM-^0t>g+X}1yVT73q zbzisKnU*c@6!U1)xbK9rn3)|+au+r}IcT)@o9!0ww41F1-fML8US%G%#Wc2kDJ(x0 zCLb3pT;oibpd4~8m2{j31)F?Xb5p*ImDsNov9GPI?MNFPr2@n(CJQG+84m$Vwfm`z z9h^v)_ww59m1^w}$1@N2sg}y(G_n>t)AQIVO1{q+7~oaeH+tk}=EeC=#{-E6d19Ry zfgEMpI5 Date: Fri, 9 Jun 2023 17:07:44 -0700 Subject: [PATCH 41/67] Add test for build task --- test/spell_check.words | 1 + test/test_pipeline.py | 136 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 128 insertions(+), 9 deletions(-) diff --git a/test/spell_check.words b/test/spell_check.words index eacc7bd..0cf49b9 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -33,6 +33,7 @@ libdir minversion mktemp namelist +noop noqa notset osfhandle diff --git a/test/test_pipeline.py b/test/test_pipeline.py index b28f159..0ac6f3f 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -2,12 +2,18 @@ # Licensed under the Apache License, Version 2.0 from copy import deepcopy +import os +from types import SimpleNamespace +from unittest.mock import patch from colcon_core.package_augmentation.python \ import PythonPackageAugmentation from colcon_core.package_descriptor import PackageDescriptor from colcon_core.package_identification.python \ import PythonPackageIdentification +from colcon_core.shell import ShellExtensionPoint +from colcon_core.task import TaskContext +from colcon_core.task.python.build import PythonBuildTask from colcon_python_project.package_augmentation.pep517 \ import PEP517PackageAugmentation from colcon_python_project.package_augmentation.pep621 \ @@ -18,6 +24,8 @@ import PEP517SetuptoolsFallbackPackageIdentification from colcon_python_project.package_identification.pep621 \ import PEP621PackageIdentification +from colcon_python_project.task.python.project.build \ + import PythonProjectBuildTask from colcon_python_setup_py.package_augmentation.python_setup_py \ import PythonPackageAugmentation as SetupPyPackageAugmentation from colcon_python_setup_py.package_identification.python_setup_py \ @@ -32,34 +40,112 @@ f'pep517.{backend}': ( backend, PEP517PackageIdentification(), - PEP517PackageAugmentation()) + PEP517PackageAugmentation(), + PythonProjectBuildTask()) for backend in MOCK_BACKENDS }, 'pep517.setuptools_fallback': ( 'legacy', PEP517SetuptoolsFallbackPackageIdentification(), - PEP517PackageAugmentation()), + PEP517PackageAugmentation(), + PythonProjectBuildTask()), 'pep621': ( 'flit', PEP621PackageIdentification(), - PEP621PackageAugmentation()), + PEP621PackageAugmentation(), + PythonProjectBuildTask()), 'setup_cfg': ( 'legacy', PythonPackageIdentification(), - PythonPackageAugmentation()), + PythonPackageAugmentation(), + PythonBuildTask()), 'setup_py': ( 'legacy', SetupPyPackageIdentification(), - SetupPyPackageAugmentation()), + SetupPyPackageAugmentation(), + PythonBuildTask()), } +class _NoEventsTaskContext(TaskContext): + + def put_event_into_queue(self, event): + pass + + +class _NoopShellExtension(ShellExtensionPoint): + + PRIORITY = 200 + + SHELL_NAME = 'noop' + + def create_prefix_script(self, prefix_path, merge_install): + return [] + + def create_package_script(self, prefix_path, pkg_name, hooks): + return [] + + def create_hook_set_value( + self, env_hook_name, prefix_path, pkg_name, name, value, + ): + hook_file = \ + prefix_path / 'share' / pkg_name / 'hook' / f'{env_hook_name}.noop' + hook_file.parent.mkdir(parents=True, exist_ok=True) + hook_file.write_text('') + return hook_file + + def create_hook_append_value( + self, env_hook_name, prefix_path, pkg_name, name, subdirectory, + ): + hook_file = \ + prefix_path / 'share' / pkg_name / 'hook' / f'{env_hook_name}.noop' + hook_file.parent.mkdir(parents=True, exist_ok=True) + hook_file.write_text('') + return hook_file + + def create_hook_prepend_value( + self, env_hook_name, prefix_path, pkg_name, name, subdirectory, + ): + hook_file = \ + prefix_path / 'share' / pkg_name / 'hook' / f'{env_hook_name}.noop' + hook_file.parent.mkdir(parents=True, exist_ok=True) + hook_file.write_text('') + return hook_file + + def create_hook_include_file( + self, env_hook_name, prefix_path, pkg_name, relative_path, + ): + hook_file = \ + prefix_path / 'share' / pkg_name / 'hook' / f'{env_hook_name}.noop' + hook_file.parent.mkdir(parents=True, exist_ok=True) + hook_file.write_text('') + return hook_file + + async def generate_command_environment( + self, task_name, build_base, dependencies, + ): + return dict(os.environ) + + +@pytest.fixture(autouse=True, scope='module') +def suppress_shell_extensions(): + with patch( + 'colcon_core.shell.get_shell_extensions', + return_value={ + _NoopShellExtension.PRIORITY: { + _NoopShellExtension.SHELL_NAME: _NoopShellExtension() + }, + }, + ): + yield + + @pytest.mark.benchmark(group='pipeline.package_identification') @pytest.mark.parametrize( - 'backend,identifier,_', + 'backend,identifier,_,__', PIPELINES.values(), ids=PIPELINES.keys()) -def test_identify(backend, identifier, _, request, benchmark): +def test_identify(backend, identifier, _, __, request, benchmark): mock_project = request.getfixturevalue(f'mock_{backend}_project') def dut(): @@ -72,10 +158,10 @@ def dut(): @pytest.mark.benchmark(group='pipeline.package_augmentation') @pytest.mark.parametrize( - 'backend,identifier,augmentor', + 'backend,identifier,augmentor,_', PIPELINES.values(), ids=PIPELINES.keys()) -def test_augment(backend, identifier, augmentor, request, benchmark): +def test_augment(backend, identifier, augmentor, _, request, benchmark): mock_project = request.getfixturevalue(f'mock_{backend}_project') mock_project = PackageDescriptor(mock_project.path) assert identifier.identify(mock_project) is None @@ -85,3 +171,35 @@ def test_augment(backend, identifier, augmentor, request, benchmark): def dut(): mock_desc = deepcopy(mock_project) assert augmentor.augment_package(mock_desc) is None + + +@pytest.mark.benchmark(group='pipeline.build') +@pytest.mark.parametrize( + 'backend,identifier,augmentor,build_task', + PIPELINES.values(), + ids=PIPELINES.keys()) +def test_build( + backend, identifier, augmentor, build_task, request, bench, + tmp_path_factory +): + mock_project = request.getfixturevalue(f'mock_{backend}_project') + mock_project = PackageDescriptor(mock_project.path) + assert identifier.identify(mock_project) is None + assert mock_project.identifies_package() + assert augmentor.augment_package(mock_project) is None + + async def dut(): + build_task.set_context(context=_NoEventsTaskContext( + pkg=mock_project, + args=SimpleNamespace( + path=str(mock_project.path), + build_base=str(tmp_path_factory.mktemp('build')), + install_base=str(tmp_path_factory.mktemp('install')), + symlink_install=False, + ), + dependencies={}, + )) + + return await build_task.build() + + assert not bench(dut) From 882a617cda384c886d244517f09880b8f5b341bd Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 9 Jun 2023 17:12:08 -0700 Subject: [PATCH 42/67] Use importlib.metadata to find and uninstall packages --- README.rst | 1 - colcon_python_project/metadata.py | 12 ++++++---- colcon_python_project/wheel.py | 38 +++++++++++++++++++++++++++++++ setup.cfg | 1 + test/spell_check.words | 1 + 5 files changed, 47 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index f563e26..b3f3aa5 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,6 @@ Extensions for `colcon-core `_ to work wi TODO ---- * Graceful and informational error handling -* Uninstall previously installed wheels before installing the newly built ones * More tests Idiosyncrasies diff --git a/colcon_python_project/metadata.py b/colcon_python_project/metadata.py index fb9d347..8bcbfdd 100644 --- a/colcon_python_project/metadata.py +++ b/colcon_python_project/metadata.py @@ -1,13 +1,17 @@ # Copyright 2022 Open Source Robotics Foundation, Inc. # Licensed under the Apache License, Version 2.0 -from email.parser import Parser from pathlib import Path from tempfile import TemporaryDirectory from colcon_python_project.hook_caller_decorator \ import get_decorated_hook_caller +try: + from importlib.metadata import Distribution +except ImportError: + from importlib_metadata import Distribution + async def load_metadata(desc): """ @@ -19,10 +23,8 @@ async def load_metadata(desc): with TemporaryDirectory() as md_dir: md_name = await hook_caller.prepare_metadata_for_build_wheel( metadata_directory=md_dir) - md_path = Path(md_dir) / md_name / 'METADATA' - with open(md_path) as f: - metadata = Parser().parse(f) - return metadata + md_path = Path(md_dir) / md_name + return Distribution.at(md_path).metadata async def load_and_cache_metadata(desc): diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index 06e31f7..99adec2 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -16,6 +16,11 @@ from colcon_core.python_install_path import get_python_install_path from distlib.scripts import ScriptMaker +try: + from importlib.metadata import Distribution +except ImportError: + from importlib_metadata import Distribution + def _get_install_path(key, install_base): return get_python_install_path(key, {'base': str(install_base)}) @@ -41,6 +46,37 @@ def write_and_record(libdir, path, lines): f'{len(raw)}') +def remove_distributions(name, install_base): + """ + Remove any installed distributions with the given name. + + :param name: Name of the distribution. + :param install_base: Path to the base directory to uninstall from. + """ + for search_path in ( + _get_install_path('purelib', install_base), + _get_install_path('platlib', install_base), + ): + dirs = set() + for dist in Distribution.discover(name=name, path=(search_path,)): + for f in dist.files: + f_abs = f.locate() + try: + f_abs.relative_to(search_path) + except ValueError: + pass + else: + f_abs.unlink() + dirs.add(f_abs.parent) + + while dirs: + d = dirs.pop() + if d == search_path or any(d.iterdir()): + continue + d.rmdir() + dirs.add(d.parent) + + def install_wheel(wheel_path, install_base, script_dir_override=None): """ Install a wheel file under the given installation base directory. @@ -61,6 +97,8 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): record_file = dist_info_dir + 'RECORD' entry_points_file = dist_info_dir + 'entry_points.txt' + remove_distributions(distribution, install_base) + with ZipFile( wheel_path, mode='r', compression=ZIP_DEFLATED, allowZip64=True ) as wf: diff --git a/setup.cfg b/setup.cfg index d8eee9a..84c4f1c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ install_requires = colcon-core>=0.12.0 colcon-ros distlib + importlib_metadata;python_version < "3.8" tomli>=1.0.0;python_version < "3.11" packages = find: zip_safe = true diff --git a/test/spell_check.words b/test/spell_check.words index 0cf49b9..39ed776 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -64,6 +64,7 @@ toml tomli tomllib traceback +uninstall unittest urlsafe zipfile From 8f94b2d4321cb7cb20833ab60f7bfa28308efe70 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 9 Jun 2023 17:39:02 -0700 Subject: [PATCH 43/67] Specifically set thread's event loop prior to benchmarking --- test/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 8c51bf3..790b148 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -69,6 +69,8 @@ def override_extensions(group_name, *args, **kwargs): @pytest.fixture def bench(benchmark, colcon_event_loop): def res(target, *args, **kwargs): + asyncio.set_event_loop(colcon_event_loop) + def dut(): return colcon_event_loop.run_until_complete( target(*args, **kwargs)) From 6e8fcd19f9d213106c95a0c8b168cbcb3d0a52b0 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Sat, 10 Jun 2023 23:57:23 -0700 Subject: [PATCH 44/67] Significantly more robust package removal --- colcon_python_project/wheel.py | 156 ++++++++++++++++++++++++++++----- test/spell_check.words | 6 ++ 2 files changed, 139 insertions(+), 23 deletions(-) diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index 99adec2..e478971 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -13,6 +13,7 @@ from zipfile import ZIP_DEFLATED from zipfile import ZipFile +from colcon_core.logging import colcon_logger from colcon_core.python_install_path import get_python_install_path from distlib.scripts import ScriptMaker @@ -21,9 +22,21 @@ except ImportError: from importlib_metadata import Distribution +logger = colcon_logger.getChild(__name__) + def _get_install_path(key, install_base): - return get_python_install_path(key, {'base': str(install_base)}) + return get_python_install_path(key, { + 'base': str(install_base), + 'platbase': str(install_base), + }) + + +def _get_script_maker(script_dir): + sm = ScriptMaker(None, script_dir) + sm.clobber = True + sm.variants = {''} + return sm def write_and_record(libdir, path, lines): @@ -46,6 +59,88 @@ def write_and_record(libdir, path, lines): f'{len(raw)}') +def enumerate_py_compiled(file): + """ + Enumerate all compiled files for a Python file. + + :param file: The path to an existing Python file. + """ + pycache = file.parent / '__pycache__' + for pyc in pycache.glob(f'{file.stem}.*.pyc'): + if pyc.is_file(): + yield pyc + + pyc = file.with_suffix('.pyc') + if pyc.is_file(): + yield pyc + + pyo = file.with_suffix('.pyo') + if pyo.is_file(): + yield pyo + + +def enumerate_files_in_distribution(dist, install_base): + """ + Enumerate all files which are part of a distribution. + + :param dist: The distribution to enumerate. + :param install_base: The installation base directory under which the + distribution is installed. + """ + # 1. Explicitly listed files + for file in dist.files: + file = file.locate().resolve() + try: + file.relative_to(install_base) + except ValueError: + pass + else: + if file.is_file(): + yield file + if file.suffix == '.py': + yield from enumerate_py_compiled(file) + + # 2. Top-level packages + for package in dist.read_text('top_level.txt').splitlines(): + file = dist.locate_file(f'{package}.py') + try: + file.relative_to(install_base) + except ValueError: + continue + if file.is_file(): + yield file + yield from enumerate_py_compiled(file) + pkgdir = dist.locate_file(package) + for file in pkgdir.rglob('*'): + if file.is_file(): + yield file + + # 3. Entry points in console_scripts + scripts = dist.entry_points.select(group='console_scripts') + if scripts: + script_dir = _get_install_path('scripts', install_base) + sm = _get_script_maker(script_dir) + for script in scripts: + for name in sm.get_script_filenames(script.name): + file = script_dir / name + if file.is_file(): + yield file + + +def enumerate_parent_dirs(file, base): + """ + Enumerate all recursive directories under a base directory to a file. + + :param file: The file under the base directory. + :param base: The base directory. + + The base directory itself is not enumerated. + """ + rel = file.parent.relative_to(base) + for i in range(1, len(rel.parts) + 1): + yield base.joinpath(*rel.parts[:i]) + + def remove_distributions(name, install_base): """ Remove any installed distributions with the given name. @@ -53,28 +148,43 @@ def remove_distributions(name, install_base): :param name: Name of the distribution. :param install_base: Path to the base directory to uninstall from. """ - for search_path in ( + libdirs = [ _get_install_path('purelib', install_base), _get_install_path('platlib', install_base), - ): - dirs = set() - for dist in Distribution.discover(name=name, path=(search_path,)): - for f in dist.files: - f_abs = f.locate() - try: - f_abs.relative_to(search_path) - except ValueError: - pass - else: - f_abs.unlink() - dirs.add(f_abs.parent) + ] - while dirs: - d = dirs.pop() - if d == search_path or any(d.iterdir()): - continue - d.rmdir() - dirs.add(d.parent) + egg_links = [] + + # If we find an egg-link, try to discover the distribution so that we + # can find and remove console scripts + for libdir in libdirs: + egg_link = libdir / f"{name.replace('_', '-')}.egg-link" + try: + with egg_link.open('r') as f: + link_dir = f.readline().rstrip() + except FileNotFoundError: + pass + else: + libdirs.append(egg_link.parent / link_dir) + egg_links.append(egg_link) + + for dist in Distribution.discover(name=name, path=libdirs): + files = set(enumerate_files_in_distribution(dist, install_base)) + parents = { + parent for file in files + for parent in enumerate_parent_dirs(file, install_base)} + for file in sorted(files): + logger.debug(f'Removing {file}') + file.unlink() + + for parent in sorted(parents, reverse=True): + if not any(parent.iterdir()): + logger.debug(f'Removing {parent}/') + parent.rmdir() + + for egg_link in sorted(egg_links): + logger.debug(f'Removing {egg_link}') + egg_link.unlink() def install_wheel(wheel_path, install_base, script_dir_override=None): @@ -97,6 +207,8 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): record_file = dist_info_dir + 'RECORD' entry_points_file = dist_info_dir + 'entry_points.txt' + install_base = Path(install_base) + remove_distributions(distribution, install_base) with ZipFile( @@ -154,9 +266,7 @@ def install_wheel(wheel_path, install_base, script_dir_override=None): script_dir = install_base / script_dir_override else: script_dir = _get_install_path('scripts', install_base) - sm = ScriptMaker(None, script_dir) - sm.clobber = True - sm.variants = {''} + sm = _get_script_maker(script_dir) specs = [ '%s = %s' % pair for pair in ep.items('console_scripts') diff --git a/test/spell_check.words b/test/spell_check.words index 39ed776..7df2c76 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -29,7 +29,9 @@ hashlib importlib importorskip iterdir +joinpath libdir +libdirs minversion mktemp namelist @@ -39,16 +41,20 @@ notset osfhandle partialmethod pathlib +pkgdir +platbase platlib plugin prepend purelib +pycache pyproject pytest pythonpath pythonwarnings relpath returncode +rglob rstrip rtype scspell From 2c04b4b14f2f031ba57faf39e0c444633d7127ed Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 12 Jun 2023 12:03:52 -0700 Subject: [PATCH 45/67] Add a slightly hacky way to remove egg-info --- colcon_python_project/wheel.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index e478971..469e45d 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -126,6 +126,25 @@ def enumerate_files_in_distribution(dist, install_base): if file.is_file(): yield file + # 4. Metadata + # TODO(cottsay): Wheels include metadata in dist.files, but that doesn't + # appear to hold for all eggs. importlib.metadata doesn't + # give us a good way to read the metadata path from a + # discovered distribution, so this is a bit of a hack. + meta_path = getattr(dist, '_path', None) + if meta_path: + try: + meta_path.relative_to(install_base) + except ValueError: + pass + else: + if meta_path.is_file(): + yield meta_path + else: + for file in meta_path.rglob('*'): + if file.is_file(): + yield file + def enumerate_parent_dirs(file, base): """ From 40a938f23e36d1c2298a93be08ef442c15ded374 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 12 Jun 2023 12:16:30 -0700 Subject: [PATCH 46/67] Add (new) usage instructions --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index b3f3aa5..9e8109d 100644 --- a/README.rst +++ b/README.rst @@ -16,3 +16,11 @@ Idiosyncrasies * For setuptools-based packages, setuptools (< 64.0.0) will leave build artifacts in the source directory. * For setuptools-based packages, symlink installs always print warnings to stderr with no good way to suppress them. * For poetry-based packages, dependencies expressed in groups (including 'test') are not discovered (use 'test' extra as a workaround). + +Using This Prototype +-------------------- + + $ mkdir -p ~/colcon_pyproject_ws/src && cd ~/colcon_pyproject_ws + $ git clone https://github.com/colcon/colcon-python-project.git -b devel src/colcon-python-project + $ colcon build + $ . install/local_setup.sh From 10cb5b1cc7ef43ffda9fcf66be4493ef41117f16 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 12 Jun 2023 12:18:21 -0700 Subject: [PATCH 47/67] ...make that a code block --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 9e8109d..2df8839 100644 --- a/README.rst +++ b/README.rst @@ -20,6 +20,8 @@ Idiosyncrasies Using This Prototype -------------------- +.. code-block:: + $ mkdir -p ~/colcon_pyproject_ws/src && cd ~/colcon_pyproject_ws $ git clone https://github.com/colcon/colcon-python-project.git -b devel src/colcon-python-project $ colcon build From d3b72c4a006b81152847c8a8cf65c5c6efc4b011 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 13 Jun 2023 10:50:21 -0700 Subject: [PATCH 48/67] Fixes for Windows --- colcon_python_project/wheel.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index 469e45d..45a295c 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -32,8 +32,8 @@ def _get_install_path(key, install_base): }) -def _get_script_maker(script_dir): - sm = ScriptMaker(None, script_dir) +def _get_script_maker(script_dir, dry_run=False): + sm = ScriptMaker(None, script_dir, dry_run=dry_run) sm.clobber = True sm.variants = {''} return sm @@ -119,12 +119,15 @@ def enumerate_files_in_distribution(dist, install_base): scripts = dist.entry_points.select(group='console_scripts') if scripts: script_dir = _get_install_path('scripts', install_base) - sm = _get_script_maker(script_dir) - for script in scripts: - for name in sm.get_script_filenames(script.name): - file = script_dir / name - if file.is_file(): - yield file + sm = _get_script_maker(script_dir, dry_run=True) + specs = [ + f'{script.name} = {script.value}' + for script in scripts + ] + for fullpath in sm.make_multiple(specs): + file = Path(fullpath) + if file.is_file(): + yield file # 4. Metadata # TODO(cottsay): Wheels include metadata in dist.files, but that doesn't @@ -172,7 +175,7 @@ def remove_distributions(name, install_base): _get_install_path('platlib', install_base), ] - egg_links = [] + egg_links = set() # If we find an egg-link, try to discover the distribution so that we # can find and remove console scripts @@ -185,7 +188,7 @@ def remove_distributions(name, install_base): pass else: libdirs.append(egg_link.parent / link_dir) - egg_links.append(egg_link) + egg_links.add(egg_link) for dist in Distribution.discover(name=name, path=libdirs): files = set(enumerate_files_in_distribution(dist, install_base)) @@ -198,7 +201,7 @@ def remove_distributions(name, install_base): for parent in sorted(parents, reverse=True): if not any(parent.iterdir()): - logger.debug(f'Removing {parent}/') + logger.debug(f'Removing {parent}' + os.path.sep) parent.rmdir() for egg_link in sorted(egg_links): From e258edb0e71954bafd656ab3def2204c664e80c0 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Sun, 18 Jun 2023 09:14:05 -0700 Subject: [PATCH 49/67] fixup! --- test/spell_check.words | 1 + 1 file changed, 1 insertion(+) diff --git a/test/spell_check.words b/test/spell_check.words index 7df2c76..6299abf 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -23,6 +23,7 @@ fdopen fdst fixturenames fsrc +fullpath functools getfixturevalue hashlib From 8357134b3131fa7f666bd27991c7ec0145d61a01 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 27 Jun 2023 14:35:13 -0700 Subject: [PATCH 50/67] Fix rebuild of packages without top_level.txt --- colcon_python_project/wheel.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py index 45a295c..80ecc11 100644 --- a/colcon_python_project/wheel.py +++ b/colcon_python_project/wheel.py @@ -101,19 +101,21 @@ def enumerate_files_in_distribution(dist, install_base): yield from enumerate_py_compiled(file) # 2. Top-level packages - for package in dist.read_text('top_level.txt').splitlines(): - file = dist.locate_file(f'{package}.py') - try: - file.relative_to(install_base) - except ValueError: - continue - if file.is_file(): - yield file - yield from enumerate_py_compiled(file) - pkgdir = dist.locate_file(package) - for file in pkgdir.rglob('*'): + top_level_text = dist.read_text('top_level.txt') + if top_level_text: + for package in top_level_text.splitlines(): + file = dist.locate_file(f'{package}.py') + try: + file.relative_to(install_base) + except ValueError: + continue if file.is_file(): yield file + yield from enumerate_py_compiled(file) + pkgdir = dist.locate_file(package) + for file in pkgdir.rglob('*'): + if file.is_file(): + yield file # 3. Entry points in console_scripts scripts = dist.entry_points.select(group='console_scripts') From 5abc1032219b89f3c062ef9687c85229cdf39980 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 17 Jul 2023 14:55:44 -0500 Subject: [PATCH 51/67] Fake setuptools options so that test verb works Partially because colcon dependencies don't carry context around what package type the dependency originated from, it's difficult for the python testing step to reliably determine the test dependencies using abstract mechanisms. For the time being, the easiest thing to do to enable those plugins for packages discovered by standards-based extensions is to mimic the setuptools metadata format for dependencies. --- colcon_python_project/metadata.py | 49 +++++++++++++++++++ .../package_augmentation/pep517.py | 9 +--- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/colcon_python_project/metadata.py b/colcon_python_project/metadata.py index 8bcbfdd..ecc85a8 100644 --- a/colcon_python_project/metadata.py +++ b/colcon_python_project/metadata.py @@ -6,6 +6,7 @@ from colcon_python_project.hook_caller_decorator \ import get_decorated_hook_caller +from distlib.util import parse_requirement try: from importlib.metadata import Distribution @@ -13,6 +14,13 @@ from importlib_metadata import Distribution +TEST_EXTRAS = ( + "'test'", '"test"', + "'tests'", '"tests"', + "'testing'", '"testing"', +) + + async def load_metadata(desc): """ Load metadata for a Python project using PEP 517. @@ -43,4 +51,45 @@ def get_metadata(): return metadata desc.metadata['get_python_project_metadata'] = get_metadata + + options = options_from_metadata(metadata) + + def get_options(_): + return options + + desc.metadata['get_python_setup_options'] = get_options return get_metadata() + + +def options_from_metadata(metadata): + """ + Extract select information from metadata in setuptools options format. + + :param metadata: The package metadata + :returns: Setuptools options + :rtype: dict + """ + install_deps = [] + test_deps = [] + + for raw_req in metadata.get_all('Requires-Dist', ()): + req = parse_requirement(raw_req) + if req.marker: + if ( + req.marker['lhs'] == 'extra' and + req.marker['op'] == '==' and + req.marker['rhs'] in TEST_EXTRAS + ): + test_deps.append(req.requirement) + else: + install_deps.append(req.requirement) + + options = {} + if install_deps: + options['install_requires'] = install_deps + if test_deps: + options['extras_require'] = { + 'test': test_deps, + } + + return options diff --git a/colcon_python_project/package_augmentation/pep517.py b/colcon_python_project/package_augmentation/pep517.py index 3d3feaf..3544abd 100644 --- a/colcon_python_project/package_augmentation/pep517.py +++ b/colcon_python_project/package_augmentation/pep517.py @@ -15,16 +15,11 @@ from colcon_python_project.hook_caller_decorator \ import get_decorated_hook_caller from colcon_python_project.metadata import load_and_cache_metadata +from colcon_python_project.metadata import TEST_EXTRAS from distlib.util import parse_requirement logger = colcon_logger.getChild(__name__) -_TEST_EXTRAS = ( - "'test'", '"test"', - "'tests'", '"tests"', - "'testing'", '"testing"', -) - class PEP517PackageAugmentation(PackageAugmentationExtensionPoint): """Augment Python packages with `pyproject.toml` using a build backend.""" @@ -84,7 +79,7 @@ def augment_package( # noqa: D102 if ( req.marker['lhs'] == 'extra' and req.marker['op'] == '==' and - req.marker['rhs'] in _TEST_EXTRAS + req.marker['rhs'] in TEST_EXTRAS ): desc.dependencies['test'].add( create_dependency_descriptor(raw_req)) From fb123bc22cb1873380250a3b21c5432c975e601c Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 28 Aug 2023 19:42:25 -0500 Subject: [PATCH 52/67] Drop partial hook methods, rally around 'call_hook' Having multiple paths to calling a specific hook has immediately backfired after a previous change started using 'call_hook' instead of the individual partial method, which routed around the setuptools decorator. It was a nice idea, but isn't really needed. --- colcon_python_project/hook_caller/__init__.py | 18 ---------- .../hook_caller_decorator/setuptools.py | 25 ++++++------- colcon_python_project/metadata.py | 4 +-- .../package_augmentation/pep517.py | 2 +- test/spell_check.words | 2 -- test/test_hooks.py | 35 ++++++++++++------- 6 files changed, 36 insertions(+), 50 deletions(-) diff --git a/colcon_python_project/hook_caller/__init__.py b/colcon_python_project/hook_caller/__init__.py index 6524c05..dd7deb8 100644 --- a/colcon_python_project/hook_caller/__init__.py +++ b/colcon_python_project/hook_caller/__init__.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 from contextlib import AbstractContextManager -from functools import partialmethod import os import pickle import sys @@ -113,23 +112,6 @@ async def call_hook(self, hook_name, **kwargs): res = pickle.load(f) return res - # PEP 517 - build_wheel = partialmethod(call_hook, 'build_wheel') - build_sdist = partialmethod(call_hook, 'build_sdist') - get_requires_for_build_wheel = partialmethod( - call_hook, 'get_requires_for_build_wheel') - prepare_metadata_for_build_wheel = partialmethod( - call_hook, 'prepare_metadata_for_build_wheel') - get_requires_for_build_sdist = partialmethod( - call_hook, 'get_requires_for_build_sdist') - - # PEP 660 - build_editable = partialmethod(call_hook, 'build_editable') - get_requires_for_build_editable = partialmethod( - call_hook, 'get_requires_for_build_editable') - prepare_metadata_for_build_editable = partialmethod( - call_hook, 'prepare_metadata_for_build_editable') - def get_hook_caller(desc, **kwargs): """ diff --git a/colcon_python_project/hook_caller_decorator/setuptools.py b/colcon_python_project/hook_caller_decorator/setuptools.py index 3a58fca..bfb836e 100644 --- a/colcon_python_project/hook_caller_decorator/setuptools.py +++ b/colcon_python_project/hook_caller_decorator/setuptools.py @@ -70,21 +70,16 @@ def __exit__(self, exc_type, exc_value, traceback): class SetuptoolsDecorator(GenericDecorator): """Enhance hooks to the setuptools build backend.""" - async def build_wheel(self, **kwargs): # noqa: D102 - config_settings = kwargs.pop('config_settings', {}) - with _ScratchEggBase(config_settings): - with _ScratchBuildBase(config_settings): - return await self._decoree.build_wheel( - config_settings=config_settings, **kwargs) - - async def get_requires_for_build_wheel(self, **kwargs): # noqa: D102 - config_settings = kwargs.pop('config_settings', {}) - with _ScratchEggBase(config_settings): - return await self._decoree.get_requires_for_build_wheel( - config_settings=config_settings, **kwargs) + async def call_hook(self, hook_name, **kwargs): # noqa: D102 + if hook_name not in ( + 'build_wheel', + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_wheel', + ): + return await self._decoree.call_hook(hook_name, **kwargs) - async def prepare_metadata_for_build_wheel(self, **kwargs): # noqa: D102 config_settings = kwargs.pop('config_settings', {}) with _ScratchEggBase(config_settings): - return await self._decoree.prepare_metadata_for_build_wheel( - config_settings=config_settings, **kwargs) + with _ScratchBuildBase(config_settings): + return await self._decoree.call_hook( + hook_name, config_settings=config_settings, **kwargs) diff --git a/colcon_python_project/metadata.py b/colcon_python_project/metadata.py index ecc85a8..b67b0c0 100644 --- a/colcon_python_project/metadata.py +++ b/colcon_python_project/metadata.py @@ -29,8 +29,8 @@ async def load_metadata(desc): """ hook_caller = get_decorated_hook_caller(desc) with TemporaryDirectory() as md_dir: - md_name = await hook_caller.prepare_metadata_for_build_wheel( - metadata_directory=md_dir) + md_name = await hook_caller.call_hook( + 'prepare_metadata_for_build_wheel', metadata_directory=md_dir) md_path = Path(md_dir) / md_name return Distribution.at(md_path).metadata diff --git a/colcon_python_project/package_augmentation/pep517.py b/colcon_python_project/package_augmentation/pep517.py index 3544abd..f4da536 100644 --- a/colcon_python_project/package_augmentation/pep517.py +++ b/colcon_python_project/package_augmentation/pep517.py @@ -46,7 +46,7 @@ def augment_package( # noqa: D102 # TODO(cottsay): get_requires_for_build_editable try: deps = loop.run_until_complete( - hook_caller.get_requires_for_build_wheel()) + hook_caller.call_hook('get_requires_for_build_wheel')) except CalledProcessError as e: logger.warn( f'An error occurred while reading metadata for {desc.name}:' diff --git a/test/spell_check.words b/test/spell_check.words index 6299abf..99bd601 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -24,7 +24,6 @@ fdst fixturenames fsrc fullpath -functools getfixturevalue hashlib importlib @@ -40,7 +39,6 @@ noop noqa notset osfhandle -partialmethod pathlib pkgdir platbase diff --git a/test/test_hooks.py b/test/test_hooks.py index ca14236..48bb7a8 100644 --- a/test/test_hooks.py +++ b/test/test_hooks.py @@ -24,8 +24,8 @@ def mock_hook_caller(mock_project): def test_build_wheel(mock_hook_caller, tmp_path_factory, bench): async def dut(): out = tmp_path_factory.mktemp('out') - return out / await mock_hook_caller.build_wheel( - wheel_directory=str(out)) + return out / await mock_hook_caller.call_hook( + 'build_wheel', wheel_directory=str(out)) wheel = bench(dut) assert wheel.is_file() @@ -34,15 +34,18 @@ async def dut(): def test_build_sdist(mock_hook_caller, tmp_path_factory, bench): async def dut(): out = tmp_path_factory.mktemp('out') - return out / await mock_hook_caller.build_sdist( - sdist_directory=str(out)) + return out / await mock_hook_caller.call_hook( + 'build_sdist', sdist_directory=str(out)) sdist = bench(dut) assert sdist.is_file() @pytest.mark.benchmark(group='hooks.get_requires_for_build_wheel') def test_get_requires_for_build_wheel(mock_hook_caller, bench): - requires = bench(mock_hook_caller.get_requires_for_build_wheel) + async def dut(): + return await mock_hook_caller.call_hook( + 'get_requires_for_build_wheel') + requires = bench(dut) assert isinstance(requires, list) @@ -52,15 +55,18 @@ def test_prepare_metadata_for_build_wheel( ): async def dut(): out = tmp_path_factory.mktemp('out') - return out / await mock_hook_caller.prepare_metadata_for_build_wheel( - metadata_directory=str(out)) + return out / await mock_hook_caller.call_hook( + 'prepare_metadata_for_build_wheel', metadata_directory=str(out)) dist_info = bench(dut) assert dist_info.is_dir() @pytest.mark.benchmark(group='hooks.get_requires_for_build_sdist') def test_get_requires_for_build_sdist(mock_hook_caller, bench): - requires = bench(mock_hook_caller.get_requires_for_build_sdist) + async def dut(): + return await mock_hook_caller.call_hook( + 'get_requires_for_build_sdist') + requires = bench(dut) assert isinstance(requires, list) @@ -71,8 +77,8 @@ def test_build_editable(mock_hook_caller, tmp_path_factory, bench): async def dut(): out = tmp_path_factory.mktemp('out') - return out / await mock_hook_caller.build_editable( - wheel_directory=str(out)) + return out / await mock_hook_caller.call_hook( + 'build_editable', wheel_directory=str(out)) wheel = bench(dut) assert wheel.is_file() @@ -82,7 +88,11 @@ def test_get_requires_for_build_editable(mock_hook_caller, bench): if mock_hook_caller.backend_name.startswith('setuptools.'): pytest.importorskip('setuptools', minversion='64.0.0') - requires = bench(mock_hook_caller.get_requires_for_build_editable) + async def dut(): + return await mock_hook_caller.call_hook( + 'get_requires_for_build_editable') + + requires = bench(dut) assert isinstance(requires, list) @@ -96,7 +106,8 @@ def test_prepare_metadata_for_build_editable( async def dut(): out = tmp_path_factory.mktemp('out') return out / \ - await mock_hook_caller.prepare_metadata_for_build_editable( + await mock_hook_caller.call_hook( + 'prepare_metadata_for_build_editable', metadata_directory=str(out)) dist_info = bench(dut) assert dist_info.is_dir() From a948e972873e2039ac9a88a55444346565e680f3 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 29 Aug 2023 09:24:01 -0500 Subject: [PATCH 53/67] Quiet pydocstyle warnings during flake8 testing This extension seems to modify the global logging config more than other extensions do, and the result is very verbose output from the flake8 tests. Suppressing pydocstyle debug messages seems to maket things reasonable again. --- test/test_flake8.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_flake8.py b/test/test_flake8.py index 2498f8b..1dbdc7b 100644 --- a/test/test_flake8.py +++ b/test/test_flake8.py @@ -11,6 +11,7 @@ # avoid debug and info messages from flake8 internals LOG.setLevel(logging.WARN) +logging.getLogger('pydocstyle').setLevel(logging.WARN) def test_flake8(): From 9beeffdb2724717c98bf207ece2deac38d561b4d Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 29 Aug 2023 11:58:01 -0500 Subject: [PATCH 54/67] Address a new deprecation in colcon-core --- colcon_python_project/argument_parser/python_project.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/colcon_python_project/argument_parser/python_project.py b/colcon_python_project/argument_parser/python_project.py index 1e09d91..7f5b76f 100644 --- a/colcon_python_project/argument_parser/python_project.py +++ b/colcon_python_project/argument_parser/python_project.py @@ -4,10 +4,15 @@ import os from colcon_core.argument_parser import ArgumentParserDecoratorExtensionPoint -from colcon_core.entry_point \ - import EXTENSION_BLOCKLIST_ENVIRONMENT_VARIABLE as BLOCK_VAR from colcon_core.plugin_system import satisfies_version +try: + from colcon_core.extension_point \ + import EXTENSION_BLOCKLIST_ENVIRONMENT_VARIABLE as BLOCK_VAR +except ImportError: + from colcon_core.entry_point \ + import EXTENSION_BLOCKLIST_ENVIRONMENT_VARIABLE as BLOCK_VAR + class PythonProjectArgumentParserDecorator( ArgumentParserDecoratorExtensionPoint From bd32ba0f0e397565dbc6f559aa1f6ee6de10fc26 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 29 Aug 2023 11:59:13 -0500 Subject: [PATCH 55/67] Fix spelling test --- test/spell_check.words | 1 + 1 file changed, 1 insertion(+) diff --git a/test/spell_check.words b/test/spell_check.words index 99bd601..5a235e6 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -47,6 +47,7 @@ plugin prepend purelib pycache +pydocstyle pyproject pytest pythonpath From 6a80a77b27d750ecd7cccdd5872e309e8a08c0c2 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 29 Aug 2023 12:47:38 -0500 Subject: [PATCH 56/67] Suppress pkg_resources deprecation in flake8_import_order --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 84c4f1c..ac3ef2c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,6 +66,7 @@ filterwarnings = error # Suppress deprecation warnings in other packages ignore:lib2to3 package is deprecated::scspell + ignore:pkg_resources is deprecated as an API::flake8_import_order ignore:SelectableGroups dict interface is deprecated::flake8 ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated::pyreadline ignore:pkg_resources is deprecated as an API::pkg_resources From 257c9fa10c32dde090993360867c500c1f75ea83 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 30 Aug 2023 11:47:23 -0500 Subject: [PATCH 57/67] Fix priority on ament package augmentation High number == run sooner, low number == run later --- .../package_augmentation/ament_python_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colcon_python_project/package_augmentation/ament_python_project.py b/colcon_python_project/package_augmentation/ament_python_project.py index cd571b5..b4d788a 100644 --- a/colcon_python_project/package_augmentation/ament_python_project.py +++ b/colcon_python_project/package_augmentation/ament_python_project.py @@ -12,7 +12,7 @@ class RosAmentPythonProjectPackageAugmentation( # Allow other augmentation extensions to perform normally and only change # the type at the end to allow the correct build extension to be invoked. - PRIORITY = 999 + PRIORITY = 5 def __init__(self): # noqa: D107 super().__init__() From ad96f10ba76281a5560af5193722965fcaf5d3cd Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 31 Aug 2023 09:27:28 -0500 Subject: [PATCH 58/67] Fix setuptools escape hatch for earlier than v64 Also add the unfortunate note about symlink installs and data_files. --- README.rst | 2 +- .../hook_caller_decorator/setuptools.py | 20 +++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 2df8839..b7de951 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,7 @@ TODO Idiosyncrasies -------------- -* For setuptools-based packages, setuptools (< 64.0.0) will leave build artifacts in the source directory. +* For setuptools-based packages, symlink installs do not install data_files. * For setuptools-based packages, symlink installs always print warnings to stderr with no good way to suppress them. * For poetry-based packages, dependencies expressed in groups (including 'test') are not discovered (use 'test' extra as a workaround). diff --git a/colcon_python_project/hook_caller_decorator/setuptools.py b/colcon_python_project/hook_caller_decorator/setuptools.py index bfb836e..8108fe4 100644 --- a/colcon_python_project/hook_caller_decorator/setuptools.py +++ b/colcon_python_project/hook_caller_decorator/setuptools.py @@ -7,6 +7,18 @@ from colcon_python_project.hook_caller_decorator import GenericDecorator from colcon_python_project.hook_caller_decorator \ import HookCallerDecoratorExtensionPoint +from packaging.version import Version + +try: + from setuptools import __version__ as setuptools_version +except ImportError: + setuptools_version = '0' + + +if Version(setuptools_version) < Version('64'): + ESCAPE_HATCH = '--global-option' +else: + ESCAPE_HATCH = '--build-option' class SetuptoolsHookCallerDecoratorExtension( @@ -37,8 +49,8 @@ def __init__(self, config_settings): def __enter__(self): temp = super().__enter__() - self._config_settings.setdefault('--build-option', []) - self._config_settings['--build-option'] += [ + self._config_settings.setdefault(ESCAPE_HATCH, []) + self._config_settings[ESCAPE_HATCH] += [ 'build', f'--build-base={temp}', ] @@ -56,8 +68,8 @@ def __init__(self, config_settings): def __enter__(self): temp = super().__enter__() - self._config_settings.setdefault('--build-option', []) - self._config_settings['--build-option'] += [ + self._config_settings.setdefault(ESCAPE_HATCH, []) + self._config_settings[ESCAPE_HATCH] += [ 'egg_info', f'--egg-base={temp}', ] From d48d6832dcbf4b56d5e97c59d2b989e2aa4e9e25 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 31 Aug 2023 14:22:01 -0500 Subject: [PATCH 59/67] Suppress an irrelevant informational message from setuptools --- colcon_python_project/hook_caller/__init__.py | 9 +++++++-- .../hook_caller_decorator/setuptools.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/colcon_python_project/hook_caller/__init__.py b/colcon_python_project/hook_caller/__init__.py index dd7deb8..e1cc92a 100644 --- a/colcon_python_project/hook_caller/__init__.py +++ b/colcon_python_project/hook_caller/__init__.py @@ -67,6 +67,11 @@ def backend_name(self): """Get the name of the backend to call hooks on.""" return self._backend_name + @property + def env(self): + """Get the environment variables to use when invoking hooks.""" + return self._env + async def list_hooks(self): """ Call into the backend to list implemented hooks. @@ -81,7 +86,7 @@ async def list_hooks(self): self._backend_name] process = await run( args, None, self._stderr_callback, - cwd=self._project_path, env=self._env, + cwd=self._project_path, env=self.env, capture_output=True) process.check_returncode() hook_names = [ @@ -105,7 +110,7 @@ async def call_hook(self, hook_name, **kwargs): have_callbacks = self._stdout_callback or self._stderr_callback process = await run( args, self._stdout_callback, self._stderr_callback, - cwd=self._project_path, env=self._env, close_fds=False, + cwd=self._project_path, env=self.env, close_fds=False, capture_output=not have_callbacks) process.check_returncode() with os.fdopen(os.dup(transport.parent_in), 'rb') as f: diff --git a/colcon_python_project/hook_caller_decorator/setuptools.py b/colcon_python_project/hook_caller_decorator/setuptools.py index 8108fe4..9fdc6d3 100644 --- a/colcon_python_project/hook_caller_decorator/setuptools.py +++ b/colcon_python_project/hook_caller_decorator/setuptools.py @@ -21,6 +21,14 @@ ESCAPE_HATCH = '--build-option' +def _add_python_warnings(env, new_warnings): + warnings = env.get('PYTHONWARNINGS', '').split(',') + for new_warning in new_warnings: + if new_warning not in warnings: + warnings.append(new_warning) + env['PYTHONWARNINGS'] = ','.join(warnings) + + class SetuptoolsHookCallerDecoratorExtension( HookCallerDecoratorExtensionPoint ): @@ -38,6 +46,10 @@ def decorate_hook_caller(self, *, hook_caller): # noqa: D102 'setuptools.build_meta:__legacy__', ): return hook_caller + _add_python_warnings(hook_caller.env, ( + 'ignore:Editable installation:setuptools.warnings.InformationOnly:' + 'setuptools.command.editable_wheel', + )) return SetuptoolsDecorator(hook_caller) From 5b2d7e43af0c87e14609b898a220b9551f5bbea5 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 5 Sep 2023 14:47:18 -0500 Subject: [PATCH 60/67] Handle hook call without explicit environment vars Fixes: d48d6832dcbf4 --- colcon_python_project/hook_caller/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colcon_python_project/hook_caller/__init__.py b/colcon_python_project/hook_caller/__init__.py index e1cc92a..0ac7241 100644 --- a/colcon_python_project/hook_caller/__init__.py +++ b/colcon_python_project/hook_caller/__init__.py @@ -58,7 +58,7 @@ def __init__( """ self._backend_name = backend_name self._project_path = str(project_path) if project_path else None - self._env = dict(env) if env else None + self._env = dict(env if env is not None else os.environ) self._stdout_callback = stdout_callback self._stderr_callback = stderr_callback From 35206d04698ecc789e1f6268962c92a9924f7178 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 24 Jun 2026 14:01:10 -0500 Subject: [PATCH 61/67] Add markers to linter tests, move linter imports Test markers can be used to easily (de-)select tests, and colcon exposes mechanisms to do so. Linters are a category of tests that are commonly called out. Additionally, if we move the imports for some of our single-purpose tests into the test function, we can avoid installing the linter dependencies entirely. This is a common case in platform packaging, where linter errors are not actionable and the dependencies are not typically installed. --- setup.cfg | 3 +++ test/spell_check.words | 1 + test/test_copyright_license.py | 7 +++++-- test/test_flake8.py | 27 ++++++++++++++------------- test/test_spell_check.py | 21 +++++++++++---------- 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/setup.cfg b/setup.cfg index ac3ef2c..65a1631 100644 --- a/setup.cfg +++ b/setup.cfg @@ -71,6 +71,9 @@ filterwarnings = ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated::pyreadline ignore:pkg_resources is deprecated as an API::pkg_resources junit_suite_name = colcon-python-project +markers = + flake8 + linter [options.entry_points] colcon_core.argument_parser = diff --git a/test/spell_check.words b/test/spell_check.words index 5a235e6..071ebfc 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -32,6 +32,7 @@ iterdir joinpath libdir libdirs +linter minversion mktemp namelist diff --git a/test/test_copyright_license.py b/test/test_copyright_license.py index e701370..dffe460 100644 --- a/test/test_copyright_license.py +++ b/test/test_copyright_license.py @@ -4,7 +4,10 @@ from pathlib import Path import sys +import pytest + +@pytest.mark.linter def test_copyright_license(): missing = check_files([Path(__file__).parents[1]]) assert not len(missing), \ @@ -25,8 +28,8 @@ def check_files(paths): if not content: continue lines = content.splitlines() - has_copyright = \ - any(line for line in lines if line.startswith('# Copyright')) + has_copyright = any(filter( + lambda line: line.startswith('# Copyright'), lines)) has_license = \ '# Licensed under the Apache License, Version 2.0' in lines if not has_copyright or not has_license: diff --git a/test/test_flake8.py b/test/test_flake8.py index 1dbdc7b..1e26cd2 100644 --- a/test/test_flake8.py +++ b/test/test_flake8.py @@ -5,16 +5,21 @@ from pathlib import Path import sys -from flake8 import LOG -from flake8.api.legacy import get_style_guide +import pytest -# avoid debug and info messages from flake8 internals -LOG.setLevel(logging.WARN) -logging.getLogger('pydocstyle').setLevel(logging.WARN) +@pytest.mark.flake8 +@pytest.mark.linter +def test_flake8(): + from flake8.api.legacy import get_style_guide + # avoid debug / info / warning messages from flake8 internals + logging.getLogger('flake8').setLevel(logging.ERROR) + + # for some reason the pydocstyle logger changes to an effective level of 1 + # set higher level to prevent the output to be flooded with debug messages + logging.getLogger('pydocstyle').setLevel(logging.WARNING) -def test_flake8(): style_guide = get_style_guide( extend_ignore=['D100', 'D104'], show_source=True, @@ -28,8 +33,7 @@ def test_flake8(): sys.stdout = sys.stderr # implicitly calls report_errors() report = style_guide.check_files([ - str(Path(__file__).parents[1] / - 'colcon_python_project'), + str(Path(__file__).parents[1] / 'colcon_python_project'), ]) report_tests = style_guide_tests.check_files([ str(Path(__file__).parents[1] / 'test'), @@ -45,9 +49,6 @@ def test_flake8(): if report_tests.total_errors: report_tests._application.formatter.show_statistics( report_tests._stats) - print( - 'flake8 reported {total_errors} errors' - .format_map(locals()), file=sys.stderr) + print(f'flake8 reported {total_errors} errors', file=sys.stderr) - assert not total_errors, \ - 'flake8 reported {total_errors} errors'.format_map(locals()) + assert not total_errors, f'flake8 reported {total_errors} errors' diff --git a/test/test_spell_check.py b/test/test_spell_check.py index 8032543..60bc823 100644 --- a/test/test_spell_check.py +++ b/test/test_spell_check.py @@ -4,25 +4,24 @@ from pathlib import Path import pytest -from scspell import Report -from scspell import SCSPELL_BUILTIN_DICT -from scspell import spell_check - spell_check_words_path = Path(__file__).parent / 'spell_check.words' @pytest.fixture(scope='module') def known_words(): - global spell_check_words_path return spell_check_words_path.read_text().splitlines() +@pytest.mark.linter def test_spell_check(known_words): + from scspell import Report + from scspell import SCSPELL_BUILTIN_DICT + from scspell import spell_check + source_filenames = [Path(__file__).parents[1] / 'setup.py'] + \ list( - (Path(__file__).parents[1] / - 'colcon_python_project') + (Path(__file__).parents[1] / 'colcon_python_project') .glob('**/*.py')) + \ list((Path(__file__).parents[1] / 'test').glob('**/*.py')) @@ -37,21 +36,23 @@ def test_spell_check(known_words): unknown_word_count = len(report.unknown_words) assert unknown_word_count == 0, \ - 'Found {unknown_word_count} unknown words: '.format_map(locals()) + \ + f'Found {unknown_word_count} unknown words: ' + \ ', '.join(sorted(report.unknown_words)) unused_known_words = set(known_words) - report.found_known_words unused_known_word_count = len(unused_known_words) assert unused_known_word_count == 0, \ - '{unused_known_word_count} words in the word list are not used: ' \ - .format_map(locals()) + ', '.join(sorted(unused_known_words)) + f'{unused_known_word_count} words in the word list are not used: ' + \ + ', '.join(sorted(unused_known_words)) +@pytest.mark.linter def test_spell_check_word_list_order(known_words): assert known_words == sorted(known_words), \ 'The word list should be ordered alphabetically' +@pytest.mark.linter def test_spell_check_word_list_duplicates(known_words): assert len(known_words) == len(set(known_words)), \ 'The word list should not contain duplicates' From 6d0c1597099a8d43c5f0ed18d811235ffca67940 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 24 Jun 2026 14:02:45 -0500 Subject: [PATCH 62/67] Switch to logger.warning instead of deprecated logger.warn --- colcon_python_project/package_identification/pep517.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colcon_python_project/package_identification/pep517.py b/colcon_python_project/package_identification/pep517.py index 0ed7326..2e1c840 100644 --- a/colcon_python_project/package_identification/pep517.py +++ b/colcon_python_project/package_identification/pep517.py @@ -56,7 +56,7 @@ def identify(self, desc): # noqa: D102 metadata = loop.run_until_complete( load_and_cache_metadata(desc)) except CalledProcessError as e: - logger.warn( + logger.warning( f'An error occurred while reading metadata for {desc.path}:' f" {e.stderr.strip().decode() or '(no output)'}") return From 4720ceabe9f892791539d879211fdc9d91055341 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 24 Jun 2026 14:05:06 -0500 Subject: [PATCH 63/67] Stop treating warnings as errors in tests While this may have been well-intended, years of builds have demonstrated that we only really see deprecation warnings in our dependencies and rarely catch anything in colcon packages. We may elect to re-enable this flag in our CI builds, but having it enabled in the package itself only makes it more difficult to maintain colcon packages downstream. --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 65a1631..0a71f5d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,7 +63,6 @@ exclude = [tool:pytest] addopts = --benchmark-disable filterwarnings = - error # Suppress deprecation warnings in other packages ignore:lib2to3 package is deprecated::scspell ignore:pkg_resources is deprecated as an API::flake8_import_order From 4eb3cddb4f4914cb7414250a7e6fe6d8e5035ef5 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 24 Jun 2026 14:03:56 -0500 Subject: [PATCH 64/67] Switch to SPDX license identifier in setup.cfg --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0a71f5d..5be9414 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,12 +13,11 @@ classifiers = Development Status :: 3 - Alpha Environment :: Plugins Intended Audience :: Developers - License :: OSI Approved :: Apache Software License Operating System :: MacOS Operating System :: POSIX Programming Language :: Python Topic :: Software Development :: Build Tools -license = Apache License, Version 2.0 +license = Apache-2.0 description = Extensions for colcon to work with Python packages which use pyproject.toml. long_description = file: README.rst keywords = colcon From d07281af65875bc3e6b8075ea170364b9e4ec2c8 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 24 Jun 2026 14:00:08 -0500 Subject: [PATCH 65/67] Add a top-level .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 4eff406..8d315d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ +/dist/ +/*.egg-info/ /.coverage +/COLCON_IGNORE +.*.swp __pycache__/ From be3e6e59c4b594dda32ef86739dcf79d48f9ecb8 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 24 Jun 2026 13:59:21 -0500 Subject: [PATCH 66/67] Update target Ubuntu/Debian suites --- publish-python.yaml | 5 ++++- stdeb.cfg | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/publish-python.yaml b/publish-python.yaml index bbe2181..a81ef3d 100644 --- a/publish-python.yaml +++ b/publish-python.yaml @@ -9,4 +9,7 @@ artifacts: repository: dirk-thomas/colcon distributions: - ubuntu:jammy - - debian:bullseye + - ubuntu:noble + - ubuntu:resolute + - debian:bookworm + - debian:trixie diff --git a/stdeb.cfg b/stdeb.cfg index fbb32c6..b00158f 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,5 +1,5 @@ [colcon-python-project] No-Python2: Depends3: python3-colcon-core (>= 0.12.0), python3-distlib, python3-tomli (>= 1.0.0) -Suite: jammy bullseye +Suite: jammy noble resolute bookworm trixie X-Python3-Version: >= 3.6 From 9d6f929aae07a61c3b2b01655cff6e4e809c0edc Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 24 Jun 2026 14:04:33 -0500 Subject: [PATCH 67/67] Add '+upstream' suffix to published deb version We continue to see interference between the deb packages we publish from the colcon project and efforts to package colcon as part of mainline Debian and Ubuntu. Using a high Debian-Version value mitigated the problems in most cases, but was not sufficient to eliminate all of the conflicts we're currently experiencing. Using a debian version suffix which falls late alphabetically appears to give our packages preference by apt. If a user enables a repository which distributes packages created by the colcon project, it is likely that they wish to use these packages instead of the ones packaged by their platform. --- stdeb.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/stdeb.cfg b/stdeb.cfg index b00158f..ea1a577 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -3,3 +3,4 @@ No-Python2: Depends3: python3-colcon-core (>= 0.12.0), python3-distlib, python3-tomli (>= 1.0.0) Suite: jammy noble resolute bookworm trixie X-Python3-Version: >= 3.6 +Upstream-Version-Suffix: +upstream