diff --git a/.gitignore b/.gitignore index c18dd8d..8d315d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ +/dist/ +/*.egg-info/ +/.coverage +/COLCON_IGNORE +.*.swp __pycache__/ diff --git a/README.rst b/README.rst index 00b6513..b7de951 100644 --- a/README.rst +++ b/README.rst @@ -5,3 +5,24 @@ Extensions for `colcon-core `_ to work wi **:warning: This is a prototype package, and isn't ready for widespread use.** **Please continue using the Python package support in** ``colcon-core`` **and** ``colcon-python-setup-py`` **until these extensions are ready.** + +TODO +---- +* Graceful and informational error handling +* More tests + +Idiosyncrasies +-------------- +* For setuptools-based packages, symlink installs do not install data_files. +* For setuptools-based packages, symlink installs always print warnings to stderr with no good way to suppress them. +* For poetry-based packages, dependencies expressed in groups (including 'test') are not discovered (use 'test' extra as a workaround). + +Using This Prototype +-------------------- + +.. code-block:: + + $ mkdir -p ~/colcon_pyproject_ws/src && cd ~/colcon_pyproject_ws + $ git clone https://github.com/colcon/colcon-python-project.git -b devel src/colcon-python-project + $ colcon build + $ . install/local_setup.sh diff --git a/colcon_python_project/argument_parser/__init__.py b/colcon_python_project/argument_parser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/argument_parser/python_project.py b/colcon_python_project/argument_parser/python_project.py new file mode 100644 index 0000000..7f5b76f --- /dev/null +++ b/colcon_python_project/argument_parser/python_project.py @@ -0,0 +1,41 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import os + +from colcon_core.argument_parser import ArgumentParserDecoratorExtensionPoint +from colcon_core.plugin_system import satisfies_version + +try: + from colcon_core.extension_point \ + import EXTENSION_BLOCKLIST_ENVIRONMENT_VARIABLE as BLOCK_VAR +except ImportError: + from colcon_core.entry_point \ + import EXTENSION_BLOCKLIST_ENVIRONMENT_VARIABLE as BLOCK_VAR + + +class PythonProjectArgumentParserDecorator( + ArgumentParserDecoratorExtensionPoint +): + """Disable extensions which conflict with colcon-python-project.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + ArgumentParserDecoratorExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + def decorate_argument_parser(self, *, parser): # noqa: D102 + blocks = [] + if os.environ.get(BLOCK_VAR.name, None): + blocks = os.environ[BLOCK_VAR.name].split(os.pathsep) + + if 'colcon_core.package_identification.python' not in blocks: + blocks.append('colcon_core.package_identification.python') + + if 'colcon_core.package_identification.python_setup_py' not in blocks: + blocks.append('colcon_core.package_identification.python_setup_py') + + os.environ[BLOCK_VAR.name] = os.pathsep.join(blocks) + + return parser diff --git a/colcon_python_project/hook_caller/__init__.py b/colcon_python_project/hook_caller/__init__.py new file mode 100644 index 0000000..0ac7241 --- /dev/null +++ b/colcon_python_project/hook_caller/__init__.py @@ -0,0 +1,138 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from contextlib import AbstractContextManager +import os +import pickle +import sys + +from colcon_core.subprocess import run +from colcon_python_project.hook_caller import _call_hook +from colcon_python_project.hook_caller import _list_hooks +from colcon_python_project.spec import load_and_cache_spec + + +class _SubprocessTransport(AbstractContextManager): + + def __enter__(self): + self.child_in, self.parent_out = os.pipe() + self.parent_in, self.child_out = os.pipe() + + try: + import msvcrt + except ImportError: + os.set_inheritable(self.child_in, True) + self.pass_in = self.child_in + os.set_inheritable(self.child_out, True) + self.pass_out = self.child_out + else: + self.pass_in = msvcrt.get_osfhandle(self.child_in) + os.set_handle_inheritable(self.pass_in, True) + self.pass_out = msvcrt.get_osfhandle(self.child_out) + os.set_handle_inheritable(self.pass_out, True) + + return self + + def __exit__(self, exc_type, exc_value, traceback): + os.close(self.parent_out) + os.close(self.parent_in) + os.close(self.child_out) + os.close(self.child_in) + + +class AsyncHookCaller: + """Calls PEP 517 style hooks asynchronously in a new process.""" + + def __init__( + self, backend_name, *, project_path=None, env=None, + stdout_callback=None, stderr_callback=None, + ): + """ + Initialize a new AsyncHookCaller. + + :param backend_name: The name of the PEP 517 build backend. + :param project_path: Path to the project's root directory. + :param env: Environment variables to use when invoking hooks. + :param stdout_callback: Callback for stdout from the hook invocation. + :param stderr_callback: Callback for stderr from the hook invocation. + """ + self._backend_name = backend_name + self._project_path = str(project_path) if project_path else None + self._env = dict(env if env is not None else os.environ) + self._stdout_callback = stdout_callback + self._stderr_callback = stderr_callback + + @property + def backend_name(self): + """Get the name of the backend to call hooks on.""" + return self._backend_name + + @property + def env(self): + """Get the environment variables to use when invoking hooks.""" + return self._env + + async def list_hooks(self): + """ + Call into the backend to list implemented hooks. + + This function lists all callable methods on the backend, which may + include more than just the hook names. + + :returns: List of hook names. + """ + args = [ + sys.executable, _list_hooks.__file__, + self._backend_name] + process = await run( + args, None, self._stderr_callback, + cwd=self._project_path, env=self.env, + capture_output=True) + process.check_returncode() + hook_names = [ + line.strip().decode() for line in process.stdout.splitlines()] + return [ + hook for hook in hook_names if hook and not hook.startswith('_')] + + async def call_hook(self, hook_name, **kwargs): + """ + Call the given hook with given arguments. + + :param hook_name: Name of the hook to call. + """ + with _SubprocessTransport() as transport: + args = [ + sys.executable, _call_hook.__file__, + self._backend_name, hook_name, + str(transport.pass_in), str(transport.pass_out)] + with os.fdopen(os.dup(transport.parent_out), 'wb') as f: + pickle.dump(kwargs, f) + have_callbacks = self._stdout_callback or self._stderr_callback + process = await run( + args, self._stdout_callback, self._stderr_callback, + cwd=self._project_path, env=self.env, close_fds=False, + capture_output=not have_callbacks) + process.check_returncode() + with os.fdopen(os.dup(transport.parent_in), 'rb') as f: + res = pickle.load(f) + return res + + +def get_hook_caller(desc, **kwargs): + """ + Create a new AsyncHookCaller instance for a package descriptor. + + :param desc: The package descriptor + """ + spec = load_and_cache_spec(desc) + backend_path = spec['build-system'].get('backend-path') + if backend_path: + # TODO(cottsay): This isn't *technically* the beginning of sys.path + # as PEP 517 calls for, but it's pretty darn close. + kwargs['env'] = dict(kwargs.get('env', os.environ)) + pythonpath = kwargs['env'].get('PYTHONPATH', '') + kwargs['env']['PYTHONPATH'] = os.pathsep.join( + backend_path + ([pythonpath] if pythonpath else [])) + return AsyncHookCaller( + spec['build-system']['build-backend'], + project_path=desc.path, **kwargs) diff --git a/colcon_python_project/hook_caller/_call_hook.py b/colcon_python_project/hook_caller/_call_hook.py new file mode 100644 index 0000000..e9e86aa --- /dev/null +++ b/colcon_python_project/hook_caller/_call_hook.py @@ -0,0 +1,29 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from importlib import import_module +import os +import pickle +import sys + + +if __name__ == '__main__': + backend_name, hook_name, child_in, child_out = sys.argv[1:] + try: + import msvcrt + except ImportError: + pass + else: + child_in = msvcrt.open_osfhandle(int(child_in), os.O_RDONLY) + child_out = msvcrt.open_osfhandle(int(child_out), 0) + if ':' in backend_name: + backend_module_name, backend_object_name = backend_name.split(':', 2) + backend_module = import_module(backend_module_name) + backend = getattr(backend_module, backend_object_name) + else: + backend = import_module(backend_name) + with os.fdopen(int(child_in), 'rb') as f: + kwargs = pickle.load(f) or {} + res = getattr(backend, hook_name)(**kwargs) + with os.fdopen(int(child_out), 'wb') as f: + pickle.dump(res, f) diff --git a/colcon_python_project/hook_caller/_list_hooks.py b/colcon_python_project/hook_caller/_list_hooks.py new file mode 100644 index 0000000..c5551e8 --- /dev/null +++ b/colcon_python_project/hook_caller/_list_hooks.py @@ -0,0 +1,19 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from importlib import import_module +import sys + + +if __name__ == '__main__': + backend_name = sys.argv[1] + if ':' in backend_name: + backend_module_name, backend_object_name = backend_name.split(':', 2) + backend_module = import_module(backend_module_name) + backend = getattr(backend_module, backend_object_name) + else: + backend = import_module(backend_name) + + for attr in dir(backend): + if callable(getattr(backend, attr)): + print(attr) diff --git a/colcon_python_project/hook_caller_decorator/__init__.py b/colcon_python_project/hook_caller_decorator/__init__.py new file mode 100644 index 0000000..a3c7a1d --- /dev/null +++ b/colcon_python_project/hook_caller_decorator/__init__.py @@ -0,0 +1,134 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import traceback + +from colcon_core.logging import colcon_logger +from colcon_core.plugin_system import instantiate_extensions +from colcon_core.plugin_system import order_extensions_by_priority +from colcon_python_project.hook_caller import get_hook_caller + +logger = colcon_logger.getChild(__name__) + + +class HookCallerDecoratorExtensionPoint: + """ + The interface for PEP 517 hook caller decorator extensions. + + For each instance the attribute `HOOK_CALLER_DECORATOR_NAME` is being + set to the basename of the entry point registering the extension. + """ + + """The version of the hook caller decorator extension interface.""" + EXTENSION_POINT_VERSION = '1.0' + + """The default priority of hook caller decorator extensions.""" + PRIORITY = 100 + + def decorate_hook_caller(self, *, hook_caller): + """ + Decorate a hook caller to perform additional functionality. + + This method must be overridden in a subclass. + + :param hook_caller: The hook caller + :returns: A decorator + """ + raise NotImplementedError() + + +def get_hook_caller_extensions(): + """ + Get the available hook caller decorator extensions. + + The extensions are ordered by their priority and entry point name. + + :rtype: OrderedDict + """ + extensions = instantiate_extensions(__name__) + for name, extension in extensions.items(): + extension.HOOK_CALLER_DECORATOR_NAME = name + return order_extensions_by_priority(extensions) + + +def decorate_hook_caller(hook_caller): + """ + Decorate the hook caller using hook caller decorator extensions. + + :param hook_caller: The hook caller + + :returns: The decorated parser + """ + extensions = get_hook_caller_extensions() + for extension in extensions.values(): + logger.log( + 1, 'decorate_hook_caller() %s', + extension.HOOK_CALLER_DECORATOR_NAME) + try: + decorated_hook_caller = extension.decorate_hook_caller( + hook_caller=hook_caller) + assert hasattr(decorated_hook_caller, 'call_hook'), \ + 'decorate_hook_caller() should return something to call hooks' + except Exception as e: # noqa: F841 + # catch exceptions raised in decorator extension + exc = traceback.format_exc() + logger.error( + 'Exception in hook caller decorator extension ' + f"'{extension.HOOK_CALLER_DECORATOR_NAME}': {e}\n{exc}") + # skip failing extension, continue with next one + else: + hook_caller = decorated_hook_caller + + return hook_caller + + +def get_decorated_hook_caller(desc, **kwargs): + """ + Create and decorate a hook caller instance for a package descriptor. + + :param desc: The package descriptor + """ + hook_caller = get_hook_caller(desc, **kwargs) + decorated_hook_caller = decorate_hook_caller(hook_caller) + return decorated_hook_caller + + +# TODO(cottsay): Promote to colcon_core and use in ArgumentParserDecorator +class GenericDecorator: + """A generic class decorator.""" + + def __init__(self, decoree, **kwargs): + """ + Create a new decorated class instance. + + :param decoree: The instance to decorate + :param **kwargs: The keyword arguments are set as attributes on this + instance + """ + assert '_decoree' not in kwargs + kwargs['_decoree'] = decoree + for k, v in kwargs.items(): + self.__dict__[k] = v + + def __getattr__(self, name): + """ + Get an attribute from this decorator if it exists or the decoree. + + :param str name: The name of the attribute + :returns: The attribute value + """ + return getattr(self.__dict__['_decoree'], name) + + def __setattr__(self, name, value): + """ + Set an attribute value on this decorator if it exists or the decoree. + + :param str name: The name of the attribute + :param value: The attribute value + """ + # overwrite existing attribute + if name in self.__dict__: + self.__dict__[name] = value + return + # set attribute on decoree + setattr(self.__dict__['_decoree'], name, value) diff --git a/colcon_python_project/hook_caller_decorator/setuptools.py b/colcon_python_project/hook_caller_decorator/setuptools.py new file mode 100644 index 0000000..9fdc6d3 --- /dev/null +++ b/colcon_python_project/hook_caller_decorator/setuptools.py @@ -0,0 +1,109 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from tempfile import TemporaryDirectory + +from colcon_core.plugin_system import satisfies_version +from colcon_python_project.hook_caller_decorator import GenericDecorator +from colcon_python_project.hook_caller_decorator \ + import HookCallerDecoratorExtensionPoint +from packaging.version import Version + +try: + from setuptools import __version__ as setuptools_version +except ImportError: + setuptools_version = '0' + + +if Version(setuptools_version) < Version('64'): + ESCAPE_HATCH = '--global-option' +else: + ESCAPE_HATCH = '--build-option' + + +def _add_python_warnings(env, new_warnings): + warnings = env.get('PYTHONWARNINGS', '').split(',') + for new_warning in new_warnings: + if new_warning not in warnings: + warnings.append(new_warning) + env['PYTHONWARNINGS'] = ','.join(warnings) + + +class SetuptoolsHookCallerDecoratorExtension( + HookCallerDecoratorExtensionPoint +): + """Decorate a hook caller for a setuptools-based backend.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + HookCallerDecoratorExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + def decorate_hook_caller(self, *, hook_caller): # noqa: D102 + if hook_caller.backend_name not in ( + 'setuptools.build_meta', + 'setuptools.build_meta:__legacy__', + ): + return hook_caller + _add_python_warnings(hook_caller.env, ( + 'ignore:Editable installation:setuptools.warnings.InformationOnly:' + 'setuptools.command.editable_wheel', + )) + return SetuptoolsDecorator(hook_caller) + + +class _ScratchBuildBase(TemporaryDirectory): + + def __init__(self, config_settings): + self._config_settings = config_settings + super().__init__() + + def __enter__(self): + temp = super().__enter__() + self._config_settings.setdefault(ESCAPE_HATCH, []) + self._config_settings[ESCAPE_HATCH] += [ + 'build', + f'--build-base={temp}', + ] + return temp + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + + +class _ScratchEggBase(TemporaryDirectory): + + def __init__(self, config_settings): + self._config_settings = config_settings + super().__init__() + + def __enter__(self): + temp = super().__enter__() + self._config_settings.setdefault(ESCAPE_HATCH, []) + self._config_settings[ESCAPE_HATCH] += [ + 'egg_info', + f'--egg-base={temp}', + ] + return temp + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + + +class SetuptoolsDecorator(GenericDecorator): + """Enhance hooks to the setuptools build backend.""" + + async def call_hook(self, hook_name, **kwargs): # noqa: D102 + if hook_name not in ( + 'build_wheel', + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_wheel', + ): + return await self._decoree.call_hook(hook_name, **kwargs) + + config_settings = kwargs.pop('config_settings', {}) + with _ScratchEggBase(config_settings): + with _ScratchBuildBase(config_settings): + return await self._decoree.call_hook( + hook_name, config_settings=config_settings, **kwargs) diff --git a/colcon_python_project/metadata.py b/colcon_python_project/metadata.py new file mode 100644 index 0000000..b67b0c0 --- /dev/null +++ b/colcon_python_project/metadata.py @@ -0,0 +1,95 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from pathlib import Path +from tempfile import TemporaryDirectory + +from colcon_python_project.hook_caller_decorator \ + import get_decorated_hook_caller +from distlib.util import parse_requirement + +try: + from importlib.metadata import Distribution +except ImportError: + from importlib_metadata import Distribution + + +TEST_EXTRAS = ( + "'test'", '"test"', + "'tests'", '"tests"', + "'testing'", '"testing"', +) + + +async def load_metadata(desc): + """ + Load metadata for a Python project using PEP 517. + + :param desc: The package descriptor + """ + hook_caller = get_decorated_hook_caller(desc) + with TemporaryDirectory() as md_dir: + md_name = await hook_caller.call_hook( + 'prepare_metadata_for_build_wheel', metadata_directory=md_dir) + md_path = Path(md_dir) / md_name + return Distribution.at(md_path).metadata + + +async def load_and_cache_metadata(desc): + """ + Get the cached metadata for a package descriptor. + + If the metadata has not been loaded yet, load and cache it. + + :param desc: The package descriptor + """ + get_metadata = desc.metadata.get('get_python_project_metadata') + if get_metadata is None: + metadata = await load_metadata(desc) + + def get_metadata(): + return metadata + + desc.metadata['get_python_project_metadata'] = get_metadata + + options = options_from_metadata(metadata) + + def get_options(_): + return options + + desc.metadata['get_python_setup_options'] = get_options + return get_metadata() + + +def options_from_metadata(metadata): + """ + Extract select information from metadata in setuptools options format. + + :param metadata: The package metadata + :returns: Setuptools options + :rtype: dict + """ + install_deps = [] + test_deps = [] + + for raw_req in metadata.get_all('Requires-Dist', ()): + req = parse_requirement(raw_req) + if req.marker: + if ( + req.marker['lhs'] == 'extra' and + req.marker['op'] == '==' and + req.marker['rhs'] in TEST_EXTRAS + ): + test_deps.append(req.requirement) + else: + install_deps.append(req.requirement) + + options = {} + if install_deps: + options['install_requires'] = install_deps + if test_deps: + options['extras_require'] = { + 'test': test_deps, + } + + return options diff --git a/colcon_python_project/package_augmentation/__init__.py b/colcon_python_project/package_augmentation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/package_augmentation/ament_python_project.py b/colcon_python_project/package_augmentation/ament_python_project.py new file mode 100644 index 0000000..b4d788a --- /dev/null +++ b/colcon_python_project/package_augmentation/ament_python_project.py @@ -0,0 +1,29 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_core.package_augmentation import PackageAugmentationExtensionPoint +from colcon_core.plugin_system import satisfies_version + + +class RosAmentPythonProjectPackageAugmentation( + PackageAugmentationExtensionPoint +): + """Convert ament_python packages to ament_python.project.""" + + # Allow other augmentation extensions to perform normally and only change + # the type at the end to allow the correct build extension to be invoked. + PRIORITY = 5 + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + def augment_package( # noqa: D102 + self, desc, *, additional_argument_names=None + ): + if desc.type != 'ros.ament_python': + return + + desc.type += '.project' diff --git a/colcon_python_project/package_augmentation/pep517.py b/colcon_python_project/package_augmentation/pep517.py new file mode 100644 index 0000000..f4da536 --- /dev/null +++ b/colcon_python_project/package_augmentation/pep517.py @@ -0,0 +1,101 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import asyncio +import logging +from subprocess import CalledProcessError + +from colcon_core.logging import colcon_logger +from colcon_core.package_augmentation \ + import PackageAugmentationExtensionPoint +from colcon_core.package_augmentation.python \ + import create_dependency_descriptor +from colcon_core.plugin_system import satisfies_version +from colcon_core.subprocess import new_event_loop +from colcon_python_project.hook_caller_decorator \ + import get_decorated_hook_caller +from colcon_python_project.metadata import load_and_cache_metadata +from colcon_python_project.metadata import TEST_EXTRAS +from distlib.util import parse_requirement + +logger = colcon_logger.getChild(__name__) + + +class PEP517PackageAugmentation(PackageAugmentationExtensionPoint): + """Augment Python packages with `pyproject.toml` using a build backend.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + # avoid debug message from asyncio when colcon uses debug log level + asyncio_logger = logging.getLogger('asyncio') + asyncio_logger.setLevel(logging.INFO) + + def augment_package( # noqa: D102 + self, desc, *, additional_argument_names=None + ): + if desc.type != 'python.project': + return + + loop = new_event_loop() + asyncio.set_event_loop(loop) + hook_caller = get_decorated_hook_caller(desc) + # TODO(cottsay): get_requires_for_build_editable + try: + deps = loop.run_until_complete( + hook_caller.call_hook('get_requires_for_build_wheel')) + except CalledProcessError as e: + logger.warn( + f'An error occurred while reading metadata for {desc.name}:' + f" {e.stderr.strip().decode() or '(no output)'}") + asyncio.set_event_loop(None) + loop.stop() + loop.close() + return + desc.dependencies.setdefault('build', set()) + desc.dependencies['build'].update( + create_dependency_descriptor(d) for d in deps) + + try: + metadata = loop.run_until_complete( + load_and_cache_metadata(desc)) + except CalledProcessError as e: + logger.warn( + f'An error occurred while reading metadata for {desc.name}:' + f" {e.stderr.strip().decode() or '(no output)'}") + return + finally: + asyncio.set_event_loop(None) + loop.stop() + loop.close() + desc.dependencies.setdefault('run', set()) + desc.dependencies.setdefault('test', set()) + for raw_req in metadata.get_all('Requires-Dist', ()): + req = parse_requirement(raw_req) + if req.marker: + if ( + req.marker['lhs'] == 'extra' and + req.marker['op'] == '==' and + req.marker['rhs'] in TEST_EXTRAS + ): + desc.dependencies['test'].add( + create_dependency_descriptor(raw_req)) + else: + desc.dependencies['run'].add( + create_dependency_descriptor(raw_req)) + + if not desc.metadata.get('version'): + desc.metadata['version'] = metadata['Version'] + + maintainers = metadata.get('Maintainer-email') + if not maintainers: + maintainers = metadata.get('Author-email') + if maintainers: + desc.metadata.setdefault('maintainers', []) + for maintainer in maintainers.split(','): + rfc822 = maintainer.strip() + if rfc822 not in desc.metadata['maintainers']: + desc.metadata['maintainers'].append(rfc822) diff --git a/colcon_python_project/package_augmentation/pep621.py b/colcon_python_project/package_augmentation/pep621.py new file mode 100644 index 0000000..eb53777 --- /dev/null +++ b/colcon_python_project/package_augmentation/pep621.py @@ -0,0 +1,67 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import logging + +from colcon_core.package_augmentation \ + import PackageAugmentationExtensionPoint +from colcon_core.package_augmentation.python import \ + create_dependency_descriptor +from colcon_core.plugin_system import satisfies_version +from colcon_python_project.spec import load_and_cache_spec + + +class PEP621PackageAugmentation(PackageAugmentationExtensionPoint): + """Augment Python packages with `pyproject.toml` using a build backend.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageAugmentationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + # avoid debug message from asyncio when colcon uses debug log level + asyncio_logger = logging.getLogger('asyncio') + asyncio_logger.setLevel(logging.INFO) + + def augment_package( # noqa: D102 + self, desc, *, additional_argument_names=None + ): + if desc.type != 'python.project': + return + + spec = load_and_cache_spec(desc) + desc.dependencies.setdefault('build', set()) + desc.dependencies['build'].update( + create_dependency_descriptor(d) + for d in spec['build-system'].get('requires') or ()) + + project = spec.get('project', {}) + desc.dependencies.setdefault('run', set()) + desc.dependencies['run'].update( + create_dependency_descriptor(d) + for d in project.get('dependencies') or ()) + + optional_deps = project.get('optional-dependencies', {}) + desc.dependencies.setdefault('test', set()) + desc.dependencies['test'].update( + create_dependency_descriptor(d) + for d in optional_deps.get('test') or ()) + + version = project.get('version') + if version: + desc.metadata['version'] = version + + maintainers = project.get('maintainers') + if not maintainers: + maintainers = project.get('authors') + if maintainers: + desc.metadata.setdefault('maintainers', []) + for entry in maintainers: + email = entry.get('email') + if not email: + continue + name = entry.get('name') + rfc822 = f'{name} <{email}>' if name else email + if rfc822 not in desc.metadata['maintainers']: + desc.metadata['maintainers'].append(rfc822) diff --git a/colcon_python_project/package_identification/__init__.py b/colcon_python_project/package_identification/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/package_identification/pep517.py b/colcon_python_project/package_identification/pep517.py new file mode 100644 index 0000000..2e1c840 --- /dev/null +++ b/colcon_python_project/package_identification/pep517.py @@ -0,0 +1,72 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import asyncio +import logging +from subprocess import CalledProcessError + +from colcon_core.logging import colcon_logger +from colcon_core.package_identification \ + import PackageIdentificationExtensionPoint +from colcon_core.plugin_system import satisfies_version +from colcon_core.subprocess import new_event_loop +from colcon_python_project.metadata import load_and_cache_metadata +from colcon_python_project.spec import SPEC_NAME + +logger = colcon_logger.getChild(__name__) + + +class PEP517PackageIdentification(PackageIdentificationExtensionPoint): + """ + Identify Python packages with `pyproject.toml` using the build backend. + + This mechanism is very slow compared with other identification extensions. + It should be able to function with any PEP 517 compliant build backend, + but those backends which see widespread use should implement a more + efficient identification extension. + """ + + # the priority needs to be higher than the extensions identifying packages + # using setup.py and/or setup.cfg files but lower than other more efficient + # PEP 517 compliant identification extensions. + PRIORITY = 110 + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageIdentificationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + # avoid debug message from asyncio when colcon uses debug log level + asyncio_logger = logging.getLogger('asyncio') + asyncio_logger.setLevel(logging.INFO) + + def identify(self, desc): # noqa: D102 + if desc.type is not None and desc.type != 'python.project': + return + + if desc.type is None: + spec_file = desc.path / SPEC_NAME + if not spec_file.is_file(): + return + + loop = new_event_loop() + asyncio.set_event_loop(loop) + try: + metadata = loop.run_until_complete( + load_and_cache_metadata(desc)) + except CalledProcessError as e: + logger.warning( + f'An error occurred while reading metadata for {desc.path}:' + f" {e.stderr.strip().decode() or '(no output)'}") + return + finally: + asyncio.set_event_loop(None) + loop.stop() + loop.close() + name = metadata.get('Name') + if not name: + return + + desc.name = name + desc.type = 'python.project' diff --git a/colcon_python_project/package_identification/pep517_setuptools_fallback.py b/colcon_python_project/package_identification/pep517_setuptools_fallback.py new file mode 100644 index 0000000..422fa5d --- /dev/null +++ b/colcon_python_project/package_identification/pep517_setuptools_fallback.py @@ -0,0 +1,41 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_python_project.package_identification.pep517 \ + import PEP517PackageIdentification + + +class PEP517SetuptoolsFallbackPackageIdentification( + PEP517PackageIdentification +): + """ + Identify legacy setuptools packages using PEP 517. + + To use this extension, disable the ``python`` and ``python_setup_py`` + identification extensions using this environment variable: + ``COLCON_EXTENSION_BLOCKLIST=colcon_core.package_identification.python:colcon_core.package_identification.python_setup_py`` + """ + + # the priority needs to be lower than all other Python package + # identification extensions. + PRIORITY = 80 + + def identify(self, desc): # noqa: D102 + if desc.type is not None and desc.type != 'python.project': + return + + setup_cfg_file = desc.path / 'setup.cfg' + setup_py_file = desc.path / 'setup.py' + + if not setup_cfg_file.is_file() and not setup_py_file.is_file(): + return + + if desc.type is not None: + return super().identify(desc) + + desc.type = 'python.project' + try: + return super().identify(desc) + finally: + if not desc.name: + desc.type = None diff --git a/colcon_python_project/package_identification/pep621.py b/colcon_python_project/package_identification/pep621.py new file mode 100644 index 0000000..7c0aa3c --- /dev/null +++ b/colcon_python_project/package_identification/pep621.py @@ -0,0 +1,44 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_core.package_identification import logger +from colcon_core.package_identification \ + import PackageIdentificationExtensionPoint +from colcon_core.plugin_system import satisfies_version +from colcon_python_project.spec import load_and_cache_spec +from colcon_python_project.spec import SPEC_NAME + + +class PEP621PackageIdentification(PackageIdentificationExtensionPoint): + """Identify Python packages with `pyproject.toml` metadata.""" + + # the priority should be higher than the extensions which enhance Python + # project package identification, and should also be hither than those + # using setup.py and/or setup.cfg files + PRIORITY = 150 + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version( + PackageIdentificationExtensionPoint.EXTENSION_POINT_VERSION, + '^1.0') + + def identify(self, desc): # noqa: D102 + if desc.type is not None and desc.type != 'python.project': + return + + spec_file = desc.path / SPEC_NAME + if not spec_file.is_file(): + return + + spec = load_and_cache_spec(desc) + name = spec.get('project', {}).get('name') + if not name: + return + + if desc.name is not None and desc.name != name: + msg = 'Package name already set to different value' + logger.error(msg) + raise RuntimeError(msg) + desc.name = name + desc.type = 'python.project' diff --git a/colcon_python_project/spec.py b/colcon_python_project/spec.py new file mode 100644 index 0000000..ab5eb6f --- /dev/null +++ b/colcon_python_project/spec.py @@ -0,0 +1,48 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +try: + from tomllib import load as toml_load +except ImportError: + from tomli import load as toml_load + + +SPEC_NAME = 'pyproject.toml' + +_DEFAULT_BUILD_SYSTEM = { + 'build-backend': 'setuptools.build_meta:__legacy__', + 'requires': ['setuptools >= 40.8.0', 'wheel'], +} + + +def load_spec(project_path): + """ + Load build system specifications for a Python project. + + :param project_path: Path to the root directory of the project + """ + spec_file = project_path / SPEC_NAME + try: + with spec_file.open('rb') as f: + spec = toml_load(f) + except FileNotFoundError: + spec = {} + + spec.setdefault('build-system', _DEFAULT_BUILD_SYSTEM) + + return spec + + +def load_and_cache_spec(desc): + """ + Get the cached spec for a package descriptor. + + If the spec has not been loaded yet, load and cache it. + + :param desc: The package descriptor + """ + spec = desc.metadata.get('python_project_spec') + if spec is None: + spec = load_spec(desc.path) + desc.metadata['python_project_spec'] = spec + return spec diff --git a/colcon_python_project/task/__init__.py b/colcon_python_project/task/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/task/ament_python/__init__.py b/colcon_python_project/task/ament_python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/task/ament_python/project/__init__.py b/colcon_python_project/task/ament_python/project/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/task/ament_python/project/build.py b/colcon_python_project/task/ament_python/project/build.py new file mode 100644 index 0000000..dac43dc --- /dev/null +++ b/colcon_python_project/task/ament_python/project/build.py @@ -0,0 +1,108 @@ +# Copyright 2016-2018 Dirk Thomas +# Licensed under the Apache License, Version 2.0 + +# Taken (nearly) verbatim from colcon_ros.task.ament_python.build + +import os +from pathlib import Path +import sys + +from colcon_core.logging import colcon_logger +from colcon_core.plugin_system import satisfies_version +from colcon_core.shell import create_environment_hook +from colcon_core.shell import get_command_environment +from colcon_core.task import create_file +from colcon_core.task import install +from colcon_core.task import TaskExtensionPoint +from colcon_core.task.python import get_data_files_mapping +from colcon_core.task.python import get_setup_data +from colcon_python_project.task.python.project.build \ + import PythonProjectBuildTask as PythonBuildTask + +logger = colcon_logger.getChild(__name__) + + +class AmentPythonProjectBuildTask(TaskExtensionPoint): + """Build ROS packages with the build type 'ament_python.project'.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version(TaskExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + async def build(self): # noqa: D102 + args = self.context.args + logger.info( + "Building ROS package in '{args.path}' with build type " + "'ament_python.project'".format_map(locals())) + + # reuse Python build task with additional logic + extension = PythonBuildTask() + extension.set_context(context=self.context) + + # additional hooks + additional_hooks = create_environment_hook( + 'ament_prefix_path', Path(args.install_base), + self.context.pkg.name, 'AMENT_PREFIX_PATH', '', mode='prepend') + + # get options from the Python manifest + try: + env = await get_command_environment( + 'setup_py', args.build_base, self.context.dependencies) + except RuntimeError as e: + logger.error(str(e)) + return 1 + setup_py_data = get_setup_data(self.context.pkg, env) + + # check if the package index and manifest are being installed + data_files = get_data_files_mapping( + setup_py_data.get('data_files', []) or []) + installs_package_index = False + installs_package_manifest = False + + # check if package index and manifest are being installed + for source, destination in data_files.items(): + if sys.platform == 'win32': + destination = Path(destination).as_posix() + # work around data files incorrectly defined as not relative + if os.path.isabs(source): + source = os.path.relpath(source, args.path) + if ( + destination == 'share/ament_index/resource_index/packages/' + + self.context.pkg.name + ): + installs_package_index = True + elif ( + source == 'package.xml' and + destination == 'share/{self.context.pkg.name}/package.xml' + .format_map(locals()) + ): + installs_package_manifest = True + + # warn about missing explicit installation + # for now implicitly install the marker and the manifest + if not installs_package_index: + # TODO remove magic helper in the future + logger.debug( + "Package '{self.context.pkg.name}' doesn't explicitly install " + 'a marker in the package index (colcon-ros currently does it ' + 'implicitly but that fallback will be removed in the future)' + .format_map(locals())) + # create package marker in ament resource index + create_file( + args, + 'share/ament_index/resource_index/packages/' + '{self.context.pkg.name}'.format_map(locals())) + if not installs_package_manifest: + # TODO remove magic helper in the future + logger.debug( + "Package '{self.context.pkg.name}' doesn't explicitly install " + "the 'package.xml' file (colcon-ros currently does it " + 'implicitly but that fallback will be removed in the future)' + .format_map(locals())) + # copy / symlink package manifest + install( + args, 'package.xml', + 'share/{self.context.pkg.name}/package.xml' + .format_map(locals())) + + return await extension.build(additional_hooks=additional_hooks) diff --git a/colcon_python_project/task/python/__init__.py b/colcon_python_project/task/python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/task/python/project/__init__.py b/colcon_python_project/task/python/project/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_python_project/task/python/project/build.py b/colcon_python_project/task/python/project/build.py new file mode 100644 index 0000000..8ec83c5 --- /dev/null +++ b/colcon_python_project/task/python/project/build.py @@ -0,0 +1,137 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from configparser import ConfigParser +import logging +import os.path +from pathlib import Path +from subprocess import CalledProcessError + +from colcon_core.environment import create_environment_hooks +from colcon_core.environment import create_environment_scripts +from colcon_core.event.output import StderrLine +from colcon_core.event.output import StdoutLine +from colcon_core.logging import colcon_logger +from colcon_core.plugin_system import satisfies_version +from colcon_core.shell import get_command_environment +from colcon_core.task import TaskExtensionPoint +from colcon_python_project.hook_caller_decorator \ + import get_decorated_hook_caller +from colcon_python_project.wheel import install_wheel + +logger = colcon_logger.getChild(__name__) + + +class PythonProjectBuildTask(TaskExtensionPoint): + """Build Python project packages.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version(TaskExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + # avoid debug message from asyncio when colcon uses debug log level + asyncio_logger = logging.getLogger('asyncio') + asyncio_logger.setLevel(logging.INFO) + + distutils_logger = logging.getLogger('distlib.util') + distutils_logger.setLevel(logging.WARN) + + async def build(self, *, additional_hooks=None): # noqa: D102 + pkg = self.context.pkg + args = self.context.args + + logger.info(f"Building Python project in '{args.path}'") + + script_dir_override = None + setup_cfg_path = pkg.path / 'setup.cfg' + if setup_cfg_path.is_file(): + parser = ConfigParser() + with setup_cfg_path.open() as f: + parser.read_file(f) + if args.symlink_install: + script_dir_override = parser.get( + 'develop', 'script-dir', fallback=None) + if not script_dir_override: + script_dir_override = parser.get( + 'develop', 'script_dir', fallback=None) + else: + script_dir_override = parser.get( + 'install', 'install-scripts', fallback=None) + if not script_dir_override: + script_dir_override = parser.get( + 'install', 'install_scripts', fallback=None) + + # Resolve setuptools-specific syntax + if script_dir_override: + _override = Path(args.install_base) + for part in Path(script_dir_override).parts: + if part == '$base': + part = args.install_base + _override /= part + script_dir_override = _override + + env = await get_command_environment( + 'python_project', args.build_base, self.context.dependencies) + + hook_caller = get_decorated_hook_caller( + pkg, env=env, stdout_callback=self._stdout_callback, + stderr_callback=self._stderr_callback) + + wheel_directory = Path(args.build_base) / 'wheel' + wheel_directory.mkdir(parents=True, exist_ok=True) + try: + build_hook_name = 'build_wheel' + if args.symlink_install: + hook_names = await hook_caller.list_hooks() + if 'build_editable' in hook_names: + build_hook_name = 'build_editable' + else: + logger.info( + f"Backend '{hook_caller.backend_name}' does not " + 'support --symlink-install - falling back to regular ' + "build for package '{pkg.name}'") + wheel_name = await hook_caller.call_hook( + build_hook_name, wheel_directory=wheel_directory) + except CalledProcessError as e: + return e.returncode + + wheel_path = wheel_directory / wheel_name + dist_info_dir = install_wheel( + wheel_path, args.install_base, + script_dir_override=script_dir_override) + + libdir = dist_info_dir.parent + records = [] + + hooks = create_environment_hooks(args.install_base, pkg.name) + records += [ + (Path(os.path.relpath(hook, libdir)).as_posix(), '', '') + for hook in hooks + ] + + scripts = create_environment_scripts( + pkg, args, default_hooks=hooks, additional_hooks=additional_hooks) + records += [ + (Path(os.path.relpath(script, libdir)).as_posix(), '', '') + for script in scripts + ] + + if build_hook_name == 'build_editable': + # PEP 610 + direct_url_json = dist_info_dir / 'direct_url.json' + with direct_url_json.open('w') as f: + f.write( + f'{{"url":"{pkg.path.absolute().as_uri()}",' + '"dir_info":{"editable":true}}\n') + records.append(( + Path(os.path.relpath(direct_url_json, libdir)).as_posix(), + '', '')) + + with (dist_info_dir / 'RECORD').open('a') as f: + f.writelines(','.join(rec) + '\n' for rec in records) + + def _stdout_callback(self, line): + self.context.put_event_into_queue(StdoutLine(line)) + + def _stderr_callback(self, line): + self.context.put_event_into_queue(StderrLine(line)) diff --git a/colcon_python_project/task/python/project/test.py b/colcon_python_project/task/python/project/test.py new file mode 100644 index 0000000..73104ca --- /dev/null +++ b/colcon_python_project/task/python/project/test.py @@ -0,0 +1,11 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from colcon_core.task.python.test import PythonTestTask + + +class PythonProjectTestTask(PythonTestTask): + """Test Python project packages.""" + + def add_arguments(self, *, parser): # noqa: D102 + pass diff --git a/colcon_python_project/wheel.py b/colcon_python_project/wheel.py new file mode 100644 index 0000000..80ecc11 --- /dev/null +++ b/colcon_python_project/wheel.py @@ -0,0 +1,308 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from base64 import urlsafe_b64encode +from configparser import ConfigParser +from email import message_from_binary_file +from hashlib import sha256 +from io import TextIOWrapper +import os.path +from pathlib import Path +import shutil +import warnings +from zipfile import ZIP_DEFLATED +from zipfile import ZipFile + +from colcon_core.logging import colcon_logger +from colcon_core.python_install_path import get_python_install_path +from distlib.scripts import ScriptMaker + +try: + from importlib.metadata import Distribution +except ImportError: + from importlib_metadata import Distribution + +logger = colcon_logger.getChild(__name__) + + +def _get_install_path(key, install_base): + return get_python_install_path(key, { + 'base': str(install_base), + 'platbase': str(install_base), + }) + + +def _get_script_maker(script_dir, dry_run=False): + sm = ScriptMaker(None, script_dir, dry_run=dry_run) + sm.clobber = True + sm.variants = {''} + return sm + + +def write_and_record(libdir, path, lines): + """ + Write file content to disk and compute a fully-qualified RECORD entry. + + :param libdir: The library directory where the package is installed. + :param path: Path to the file to be written, either absolute or + relative to the library directory. + :param lines: Enumerable of lines of text to be written. + :returns: Three-element tuple constituting the file's RECORD entry. + """ + path = libdir / path + raw = (os.linesep.join(lines) + os.linesep).encode() + digest = urlsafe_b64encode(sha256(raw).digest()).rstrip(b'=').decode() + path.write_bytes(raw) + return ( + Path(os.path.relpath(path, libdir)).as_posix(), + f'sha256={digest}', + f'{len(raw)}') + + +def enumerate_py_compiled(file): + """ + Enumerate all compiled files for a Python file. + + :param file: The path to an existing Python file. + """ + pycache = file.parent / '__pycache__' + for pyc in pycache.glob(f'{file.stem}.*.pyc'): + if pyc.is_file(): + yield pyc + + pyc = file.with_suffix('.pyc') + if pyc.is_file(): + yield pyc + + pyo = file.with_suffix('.pyo') + if pyo.is_file(): + yield pyo + + +def enumerate_files_in_distribution(dist, install_base): + """ + Enumerate all files which are part of a distribution. + + :param dist: The distribution to enumerate. + :param install_base: The installation base directory under which the + distribution is installed. + """ + # 1. Explicitly listed files + for file in dist.files: + file = file.locate().resolve() + try: + file.relative_to(install_base) + except ValueError: + pass + else: + if file.is_file(): + yield file + if file.suffix == '.py': + yield from enumerate_py_compiled(file) + + # 2. Top-level packages + top_level_text = dist.read_text('top_level.txt') + if top_level_text: + for package in top_level_text.splitlines(): + file = dist.locate_file(f'{package}.py') + try: + file.relative_to(install_base) + except ValueError: + continue + if file.is_file(): + yield file + yield from enumerate_py_compiled(file) + pkgdir = dist.locate_file(package) + for file in pkgdir.rglob('*'): + if file.is_file(): + yield file + + # 3. Entry points in console_scripts + scripts = dist.entry_points.select(group='console_scripts') + if scripts: + script_dir = _get_install_path('scripts', install_base) + sm = _get_script_maker(script_dir, dry_run=True) + specs = [ + f'{script.name} = {script.value}' + for script in scripts + ] + for fullpath in sm.make_multiple(specs): + file = Path(fullpath) + if file.is_file(): + yield file + + # 4. Metadata + # TODO(cottsay): Wheels include metadata in dist.files, but that doesn't + # appear to hold for all eggs. importlib.metadata doesn't + # give us a good way to read the metadata path from a + # discovered distribution, so this is a bit of a hack. + meta_path = getattr(dist, '_path', None) + if meta_path: + try: + meta_path.relative_to(install_base) + except ValueError: + pass + else: + if meta_path.is_file(): + yield meta_path + else: + for file in meta_path.rglob('*'): + if file.is_file(): + yield file + + +def enumerate_parent_dirs(file, base): + """ + Enumerate all recursive directories under a base directory to a file. + + :param file: The file under the base directory. + :param base: The base directory. + + The base directory itself is not enumerated. + """ + rel = file.parent.relative_to(base) + for i in range(1, len(rel.parts) + 1): + yield base.joinpath(*rel.parts[:i]) + + +def remove_distributions(name, install_base): + """ + Remove any installed distributions with the given name. + + :param name: Name of the distribution. + :param install_base: Path to the base directory to uninstall from. + """ + libdirs = [ + _get_install_path('purelib', install_base), + _get_install_path('platlib', install_base), + ] + + egg_links = set() + + # If we find an egg-link, try to discover the distribution so that we + # can find and remove console scripts + for libdir in libdirs: + egg_link = libdir / f"{name.replace('_', '-')}.egg-link" + try: + with egg_link.open('r') as f: + link_dir = f.readline().rstrip() + except FileNotFoundError: + pass + else: + libdirs.append(egg_link.parent / link_dir) + egg_links.add(egg_link) + + for dist in Distribution.discover(name=name, path=libdirs): + files = set(enumerate_files_in_distribution(dist, install_base)) + parents = { + parent for file in files + for parent in enumerate_parent_dirs(file, install_base)} + for file in sorted(files): + logger.debug(f'Removing {file}') + file.unlink() + + for parent in sorted(parents, reverse=True): + if not any(parent.iterdir()): + logger.debug(f'Removing {parent}' + os.path.sep) + parent.rmdir() + + for egg_link in sorted(egg_links): + logger.debug(f'Removing {egg_link}') + egg_link.unlink() + + +def install_wheel(wheel_path, install_base, script_dir_override=None): + """ + Install a wheel file under the given installation base directory. + + :param wheel_path: Path to the wheel file to be installed. + :param install_base: Path to the base directory to install under. + :param script_dir_override: Override the default script install + directory + :returns: Path to the installed distribution info directory + """ + wheel_name = wheel_path.name.split('-') + if len(wheel_name) not in (5, 6): + raise RuntimeError('Invalid wheel file name') + distribution, version = wheel_name[:2] + dist_info_dir = f'{distribution}-{version}.dist-info/' + data_dir = f'{distribution}-{version}.data/' + wheel_file = dist_info_dir + 'WHEEL' + record_file = dist_info_dir + 'RECORD' + entry_points_file = dist_info_dir + 'entry_points.txt' + + install_base = Path(install_base) + + remove_distributions(distribution, install_base) + + with ZipFile( + wheel_path, mode='r', compression=ZIP_DEFLATED, allowZip64=True + ) as wf: + with wf.open(wheel_file) as wf_mf: + wheel_metadata = message_from_binary_file(wf_mf) + + wheel_version = wheel_metadata.get('Wheel-Version', '').split('.') + if len(wheel_version) < 2 or wheel_version[0] != '1': + raise RuntimeError('Wheel file is not supported') + elif wheel_version[1] != '0': + warnings.warn('Wheel format is newer than supported version') + + if wheel_metadata.get('Root-Is-Purelib') in ('true',): + libdir = _get_install_path('purelib', install_base) + else: + libdir = _get_install_path('platlib', install_base) + + records = [] + with wf.open(record_file) as wf_rec_bin: + with TextIOWrapper(wf_rec_bin) as wf_rec: + for line in wf_rec: + if ',' in line: + records.append(line.strip().split(',')) + + for record in records: + if record[0] == record_file: + continue + elif not record[0].startswith(data_dir): + wf.extract(record[0], libdir) + continue + + _, key, subpath = record[0].split('/', 2) + target = Path(_get_install_path(key, install_base)) + target /= subpath + target.parent.mkdir(parents=True, exist_ok=True) + with wf.open(record[0]) as fsrc: + with target.open('wb') as fdst: + shutil.copyfileobj(fsrc, fdst) + record[0] = os.path.relpath(target, start=libdir) + + records.append(write_and_record( + libdir, + dist_info_dir + 'INSTALLER', + ('colcon-python-project',))) + + if entry_points_file in wf.namelist(): + ep = ConfigParser() + with wf.open(entry_points_file) as wf_ep_bin: + with TextIOWrapper(wf_ep_bin) as wf_ep: + ep.read_file(wf_ep) + if ep.has_section('console_scripts'): + if script_dir_override: + script_dir = install_base / script_dir_override + else: + script_dir = _get_install_path('scripts', install_base) + sm = _get_script_maker(script_dir) + specs = [ + '%s = %s' % pair + for pair in ep.items('console_scripts') + ] + scripts_made = sm.make_multiple(specs) + + records += [ + (Path(os.path.relpath(s, libdir)).as_posix(), '', '') + for s in scripts_made + ] + + with (libdir / record_file).open('w') as f: + f.writelines(','.join(rec) + '\n' for rec in records) + + return libdir / dist_info_dir diff --git a/publish-python.yaml b/publish-python.yaml index bbe2181..a81ef3d 100644 --- a/publish-python.yaml +++ b/publish-python.yaml @@ -9,4 +9,7 @@ artifacts: repository: dirk-thomas/colcon distributions: - ubuntu:jammy - - debian:bullseye + - ubuntu:noble + - ubuntu:resolute + - debian:bookworm + - debian:trixie diff --git a/setup.cfg b/setup.cfg index 432bfd1..5be9414 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,19 +13,22 @@ classifiers = Development Status :: 3 - Alpha Environment :: Plugins Intended Audience :: Developers - License :: OSI Approved :: Apache Software License Operating System :: MacOS Operating System :: POSIX Programming Language :: Python Topic :: Software Development :: Build Tools -license = Apache License, Version 2.0 +license = Apache-2.0 description = Extensions for colcon to work with Python packages which use pyproject.toml. long_description = file: README.rst keywords = colcon [options] install_requires = - colcon-core + colcon-core>=0.12.0 + colcon-ros + distlib + importlib_metadata;python_version < "3.8" + tomli>=1.0.0;python_version < "3.11" packages = find: zip_safe = true @@ -40,25 +43,68 @@ test = flake8-docstrings flake8-import-order flake8-quotes + flit pep8-naming + poetry pylint pytest + pytest-benchmark pytest-cov scspell3k>=2.2 + setuptools>=40.8.0 + wheel + +[options.packages.find] +exclude = + test + test.* [tool:pytest] +addopts = --benchmark-disable filterwarnings = - error # Suppress deprecation warnings in other packages ignore:lib2to3 package is deprecated::scspell + ignore:pkg_resources is deprecated as an API::flake8_import_order ignore:SelectableGroups dict interface is deprecated::flake8 ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated::pyreadline + ignore:pkg_resources is deprecated as an API::pkg_resources junit_suite_name = colcon-python-project +markers = + flake8 + linter [options.entry_points] +colcon_core.argument_parser = + python_project = colcon_python_project.argument_parser.python_project:PythonProjectArgumentParserDecorator +colcon_core.extension_point = + colcon_python_project.hook_caller_decorator = colcon_python_project.hook_caller_decorator:HookCallerDecoratorExtensionPoint +colcon_core.package_augmentation = + ament_python_project = colcon_python_project.package_augmentation.ament_python_project:RosAmentPythonProjectPackageAugmentation + pep517 = colcon_python_project.package_augmentation.pep517:PEP517PackageAugmentation + pep621 = colcon_python_project.package_augmentation.pep621:PEP621PackageAugmentation +colcon_core.package_identification = + pep517 = colcon_python_project.package_identification.pep517:PEP517PackageIdentification + pep517_setuptools_fallback = colcon_python_project.package_identification.pep517_setuptools_fallback:PEP517SetuptoolsFallbackPackageIdentification + pep621 = colcon_python_project.package_identification.pep621:PEP621PackageIdentification +colcon_core.task.build = + python.project = colcon_python_project.task.python.project.build:PythonProjectBuildTask + ros.ament_python.project = colcon_python_project.task.ament_python.project.build:AmentPythonProjectBuildTask +colcon_core.task.test = + python.project = colcon_python_project.task.python.project.test:PythonProjectTestTask + ros.ament_python.project = colcon_ros.task.ament_python.test:AmentPythonTestTask +colcon_python_project.hook_caller_decorator = + setuptools = colcon_python_project.hook_caller_decorator.setuptools:SetuptoolsHookCallerDecoratorExtension [flake8] import-order-style = google [coverage:run] source = colcon_python_project +omit = + setup.py + test/* + +[coverage:report] +omit = + setup.py + test/* diff --git a/stdeb.cfg b/stdeb.cfg index ba1b742..ea1a577 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,5 +1,6 @@ [colcon-python-project] No-Python2: -Depends3: python3-colcon-core -Suite: jammy bullseye +Depends3: python3-colcon-core (>= 0.12.0), python3-distlib, python3-tomli (>= 1.0.0) +Suite: jammy noble resolute bookworm trixie X-Python3-Version: >= 3.6 +Upstream-Version-Suffix: +upstream diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/backend_fixtures/__init__.py b/test/backend_fixtures/__init__.py new file mode 100644 index 0000000..1b2f3d7 --- /dev/null +++ b/test/backend_fixtures/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from .flit import mock_backend_project as mock_flit_project # noqa: F401 +from .legacy import mock_backend_project as mock_legacy_project # noqa: F401 +from .poetry import mock_backend_project as mock_poetry_project # noqa: F401 +from .setuptools \ + import mock_backend_project as mock_setuptools_project # noqa: F401 + + +MOCK_BACKENDS = ('flit', 'legacy', 'poetry', 'setuptools') diff --git a/test/backend_fixtures/flit.py b/test/backend_fixtures/flit.py new file mode 100644 index 0000000..1ca9315 --- /dev/null +++ b/test/backend_fixtures/flit.py @@ -0,0 +1,23 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import pytest + + +@pytest.fixture +def mock_backend_project(mock_desc): + with (mock_desc.path / 'pyproject.toml').open('w') as f: + f.write('\n'.join([ + '[build-system]', + 'requires = ["flit_core >=3.2,<4"]', + 'build-backend = "flit_core.buildapi"', + '', + '[project]', + f'name = "{mock_desc.name}"', + 'version = "0.0.0"', + 'description = "A test project"', + '', + '[tool.flit.module]', + 'name = "test_project"', + ])) + return mock_desc diff --git a/test/backend_fixtures/legacy.py b/test/backend_fixtures/legacy.py new file mode 100644 index 0000000..27a9cac --- /dev/null +++ b/test/backend_fixtures/legacy.py @@ -0,0 +1,27 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import pytest + + +@pytest.fixture +def mock_backend_project(mock_desc): + with (mock_desc.path / 'setup.cfg').open('w') as f: + f.write('\n'.join([ + '[metadata]', + f'name = {mock_desc.name}', + '[options]', + 'packages = find:', + ])) + with (mock_desc.path / 'setup.py').open('w') as f: + f.write('\n'.join([ + 'from setuptools import setup', + 'setup()', + ])) + with (mock_desc.path / 'pyproject.toml').open('w') as f: + f.write('\n'.join([ + '[build-system]', + 'requires = ["setuptools>=40.8.0"]', + 'build-backend = "setuptools.build_meta:__legacy__"', + ])) + return mock_desc diff --git a/test/backend_fixtures/poetry.py b/test/backend_fixtures/poetry.py new file mode 100644 index 0000000..feadd4b --- /dev/null +++ b/test/backend_fixtures/poetry.py @@ -0,0 +1,21 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import pytest + + +@pytest.fixture +def mock_backend_project(mock_desc): + with (mock_desc.path / 'pyproject.toml').open('w') as f: + f.write('\n'.join([ + '[build-system]', + 'requires = ["poetry-core"]', + 'build-backend = "poetry.core.masonry.api"', + '', + '[tool.poetry]', + f'name = "{mock_desc.name}"', + 'version = "0.0.0"', + 'description = "A test project"', + 'authors = ["Author "]', + ])) + return mock_desc diff --git a/test/backend_fixtures/setuptools.py b/test/backend_fixtures/setuptools.py new file mode 100644 index 0000000..65e3ae2 --- /dev/null +++ b/test/backend_fixtures/setuptools.py @@ -0,0 +1,20 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import pytest + + +@pytest.fixture +def mock_backend_project(mock_desc): + with (mock_desc.path / 'pyproject.toml').open('w') as f: + f.write('\n'.join([ + '[build-system]', + 'requires = ["setuptools>=61.0.0"]', + 'build-backend = "setuptools.build_meta"', + '', + '[project]', + f'name = "{mock_desc.name}"', + 'version = "0.0.0"', + 'description = "A test project"', + ])) + return mock_desc diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..790b148 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,97 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import asyncio +import logging +import os +import unittest.mock +import warnings + +from colcon_core.package_descriptor import PackageDescriptor +from colcon_core.subprocess import new_event_loop +from colcon_python_project.hook_caller_decorator.setuptools \ + import SetuptoolsHookCallerDecoratorExtension +import pytest + +from .backend_fixtures import * # noqa: F401, F403 +from .backend_fixtures import MOCK_BACKENDS + + +@pytest.fixture +def colcon_event_loop(): + loop = new_event_loop() + asyncio.set_event_loop(loop) + yield loop + loop.run_until_complete(loop.shutdown_asyncgens()) + asyncio.set_event_loop(None) + loop.close() + + +@pytest.fixture(autouse=True) +def better_benchmarking(request): + if 'benchmark' not in request.fixturenames: + yield + return + benchmark = request.getfixturevalue('benchmark') + if not benchmark.enabled: + yield + return + if 'COV_CORE_DATAFILE' in os.environ: + warnings.warn( + "Can't run benchmarks with coverage enabled. " + 'Re-run with --no-cov.') + benchmark.disabled = True + yield + return + logging.disable(level=logging.ERROR) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + yield + logging.disable(level=logging.NOTSET) + + +@pytest.fixture(autouse=True, scope='session') +def static_extensions(): + def override_extensions(group_name, *args, **kwargs): + if group_name == 'colcon_python_project.hook_caller_decorator': + return { + 'setuptools': SetuptoolsHookCallerDecoratorExtension(), + } + return unittest.mock.DEFAULT + + with unittest.mock.patch( + 'colcon_python_project.hook_caller_decorator.instantiate_extensions', + side_effect=override_extensions, + ) as p: + yield p + + +@pytest.fixture +def bench(benchmark, colcon_event_loop): + def res(target, *args, **kwargs): + asyncio.set_event_loop(colcon_event_loop) + + def dut(): + return colcon_event_loop.run_until_complete( + target(*args, **kwargs)) + return benchmark(dut) + return res + + +@pytest.fixture +def mock_desc(tmp_path): + src = tmp_path / 'src' + src.mkdir() + with (src / 'test_project.py').open('w'): + pass + with (src / 'README').open('w'): + pass + desc = PackageDescriptor(src) + desc.name = 'test-project' + desc.type = 'python.project' + return desc + + +@pytest.fixture(params=MOCK_BACKENDS) +def mock_project(request): + return request.getfixturevalue(f'mock_{request.param}_project') diff --git a/test/spell_check.words b/test/spell_check.words index 91e2768..071ebfc 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -1,8 +1,77 @@ +ament apache +asyncgens +asyncio +augmentor +autouse +backend +backends +blocklist +buildapi colcon +configparser +conftest +contextlib +copyfileobj +cottsay +datafile +decoree +deepcopy +deps +distlib +fdopen +fdst +fixturenames +fsrc +fullpath +getfixturevalue +hashlib +importlib +importorskip iterdir +joinpath +libdir +libdirs +linter +minversion +mktemp +namelist +noop +noqa +notset +osfhandle pathlib +pkgdir +platbase +platlib +plugin +prepend +purelib +pycache +pydocstyle +pyproject pytest +pythonpath +pythonwarnings +relpath +returncode +rglob +rstrip +rtype scspell +sdist setuptools +simplefilter +subpath +symlink +tempfile thomas +todo +toml +tomli +tomllib +traceback +uninstall +unittest +urlsafe +zipfile diff --git a/test/test_copyright_license.py b/test/test_copyright_license.py index e701370..dffe460 100644 --- a/test/test_copyright_license.py +++ b/test/test_copyright_license.py @@ -4,7 +4,10 @@ from pathlib import Path import sys +import pytest + +@pytest.mark.linter def test_copyright_license(): missing = check_files([Path(__file__).parents[1]]) assert not len(missing), \ @@ -25,8 +28,8 @@ def check_files(paths): if not content: continue lines = content.splitlines() - has_copyright = \ - any(line for line in lines if line.startswith('# Copyright')) + has_copyright = any(filter( + lambda line: line.startswith('# Copyright'), lines)) has_license = \ '# Licensed under the Apache License, Version 2.0' in lines if not has_copyright or not has_license: diff --git a/test/test_flake8.py b/test/test_flake8.py index 2498f8b..1e26cd2 100644 --- a/test/test_flake8.py +++ b/test/test_flake8.py @@ -5,15 +5,21 @@ from pathlib import Path import sys -from flake8 import LOG -from flake8.api.legacy import get_style_guide +import pytest -# avoid debug and info messages from flake8 internals -LOG.setLevel(logging.WARN) +@pytest.mark.flake8 +@pytest.mark.linter +def test_flake8(): + from flake8.api.legacy import get_style_guide + # avoid debug / info / warning messages from flake8 internals + logging.getLogger('flake8').setLevel(logging.ERROR) + + # for some reason the pydocstyle logger changes to an effective level of 1 + # set higher level to prevent the output to be flooded with debug messages + logging.getLogger('pydocstyle').setLevel(logging.WARNING) -def test_flake8(): style_guide = get_style_guide( extend_ignore=['D100', 'D104'], show_source=True, @@ -27,8 +33,7 @@ def test_flake8(): sys.stdout = sys.stderr # implicitly calls report_errors() report = style_guide.check_files([ - str(Path(__file__).parents[1] / - 'colcon_python_project'), + str(Path(__file__).parents[1] / 'colcon_python_project'), ]) report_tests = style_guide_tests.check_files([ str(Path(__file__).parents[1] / 'test'), @@ -44,9 +49,6 @@ def test_flake8(): if report_tests.total_errors: report_tests._application.formatter.show_statistics( report_tests._stats) - print( - 'flake8 reported {total_errors} errors' - .format_map(locals()), file=sys.stderr) + print(f'flake8 reported {total_errors} errors', file=sys.stderr) - assert not total_errors, \ - 'flake8 reported {total_errors} errors'.format_map(locals()) + assert not total_errors, f'flake8 reported {total_errors} errors' diff --git a/test/test_hooks.py b/test/test_hooks.py new file mode 100644 index 0000000..48bb7a8 --- /dev/null +++ b/test/test_hooks.py @@ -0,0 +1,121 @@ +# Copyright 2022 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +import os +import sys + +from colcon_python_project.hook_caller_decorator \ + import get_decorated_hook_caller +import pytest + + +@pytest.fixture +def mock_hook_caller(mock_project): + env = { + **os.environ, + 'PYTHONWARNINGS': 'ignore', + } + return get_decorated_hook_caller( + mock_project, + env=env, stderr_callback=sys.stderr.buffer.write) + + +@pytest.mark.benchmark(group='hooks.build_wheel') +def test_build_wheel(mock_hook_caller, tmp_path_factory, bench): + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / await mock_hook_caller.call_hook( + 'build_wheel', wheel_directory=str(out)) + wheel = bench(dut) + assert wheel.is_file() + + +@pytest.mark.benchmark(group='hooks.build_sdist') +def test_build_sdist(mock_hook_caller, tmp_path_factory, bench): + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / await mock_hook_caller.call_hook( + 'build_sdist', sdist_directory=str(out)) + sdist = bench(dut) + assert sdist.is_file() + + +@pytest.mark.benchmark(group='hooks.get_requires_for_build_wheel') +def test_get_requires_for_build_wheel(mock_hook_caller, bench): + async def dut(): + return await mock_hook_caller.call_hook( + 'get_requires_for_build_wheel') + requires = bench(dut) + assert isinstance(requires, list) + + +@pytest.mark.benchmark(group='hooks.prepare_metadata_for_build_wheel') +def test_prepare_metadata_for_build_wheel( + mock_hook_caller, tmp_path_factory, bench +): + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / await mock_hook_caller.call_hook( + 'prepare_metadata_for_build_wheel', metadata_directory=str(out)) + dist_info = bench(dut) + assert dist_info.is_dir() + + +@pytest.mark.benchmark(group='hooks.get_requires_for_build_sdist') +def test_get_requires_for_build_sdist(mock_hook_caller, bench): + async def dut(): + return await mock_hook_caller.call_hook( + 'get_requires_for_build_sdist') + requires = bench(dut) + assert isinstance(requires, list) + + +@pytest.mark.benchmark(group='hooks.build_editable') +def test_build_editable(mock_hook_caller, tmp_path_factory, bench): + if mock_hook_caller.backend_name.startswith('setuptools.'): + pytest.importorskip('setuptools', minversion='64.0.0') + + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / await mock_hook_caller.call_hook( + 'build_editable', wheel_directory=str(out)) + wheel = bench(dut) + assert wheel.is_file() + + +@pytest.mark.benchmark(group='hooks.get_requires_for_build_editable') +def test_get_requires_for_build_editable(mock_hook_caller, bench): + if mock_hook_caller.backend_name.startswith('setuptools.'): + pytest.importorskip('setuptools', minversion='64.0.0') + + async def dut(): + return await mock_hook_caller.call_hook( + 'get_requires_for_build_editable') + + requires = bench(dut) + assert isinstance(requires, list) + + +@pytest.mark.benchmark(group='hooks.prepare_metadata_for_build_editable') +def test_prepare_metadata_for_build_editable( + mock_hook_caller, tmp_path_factory, bench +): + if mock_hook_caller.backend_name.startswith('setuptools.'): + pytest.importorskip('setuptools', minversion='64.0.0') + + async def dut(): + out = tmp_path_factory.mktemp('out') + return out / \ + await mock_hook_caller.call_hook( + 'prepare_metadata_for_build_editable', + metadata_directory=str(out)) + dist_info = bench(dut) + assert dist_info.is_dir() + + +@pytest.mark.benchmark(group='hooks.list_hooks') +def test_list_hooks(mock_hook_caller, bench): + hook_names = bench(mock_hook_caller.list_hooks) + assert isinstance(hook_names, list) + assert 'build_sdist' in hook_names + assert 'build_wheel' in hook_names diff --git a/test/test_pipeline.py b/test/test_pipeline.py new file mode 100644 index 0000000..0ac6f3f --- /dev/null +++ b/test/test_pipeline.py @@ -0,0 +1,205 @@ +# Copyright 2023 Open Source Robotics Foundation, Inc. +# Licensed under the Apache License, Version 2.0 + +from copy import deepcopy +import os +from types import SimpleNamespace +from unittest.mock import patch + +from colcon_core.package_augmentation.python \ + import PythonPackageAugmentation +from colcon_core.package_descriptor import PackageDescriptor +from colcon_core.package_identification.python \ + import PythonPackageIdentification +from colcon_core.shell import ShellExtensionPoint +from colcon_core.task import TaskContext +from colcon_core.task.python.build import PythonBuildTask +from colcon_python_project.package_augmentation.pep517 \ + import PEP517PackageAugmentation +from colcon_python_project.package_augmentation.pep621 \ + import PEP621PackageAugmentation +from colcon_python_project.package_identification.pep517 \ + import PEP517PackageIdentification +from colcon_python_project.package_identification.pep517_setuptools_fallback \ + import PEP517SetuptoolsFallbackPackageIdentification +from colcon_python_project.package_identification.pep621 \ + import PEP621PackageIdentification +from colcon_python_project.task.python.project.build \ + import PythonProjectBuildTask +from colcon_python_setup_py.package_augmentation.python_setup_py \ + import PythonPackageAugmentation as SetupPyPackageAugmentation +from colcon_python_setup_py.package_identification.python_setup_py \ + import PythonPackageIdentification as SetupPyPackageIdentification +import pytest + +from .conftest import MOCK_BACKENDS + + +PIPELINES = { + **{ + f'pep517.{backend}': ( + backend, + PEP517PackageIdentification(), + PEP517PackageAugmentation(), + PythonProjectBuildTask()) + for backend in MOCK_BACKENDS + }, + 'pep517.setuptools_fallback': ( + 'legacy', + PEP517SetuptoolsFallbackPackageIdentification(), + PEP517PackageAugmentation(), + PythonProjectBuildTask()), + 'pep621': ( + 'flit', + PEP621PackageIdentification(), + PEP621PackageAugmentation(), + PythonProjectBuildTask()), + 'setup_cfg': ( + 'legacy', + PythonPackageIdentification(), + PythonPackageAugmentation(), + PythonBuildTask()), + 'setup_py': ( + 'legacy', + SetupPyPackageIdentification(), + SetupPyPackageAugmentation(), + PythonBuildTask()), +} + + +class _NoEventsTaskContext(TaskContext): + + def put_event_into_queue(self, event): + pass + + +class _NoopShellExtension(ShellExtensionPoint): + + PRIORITY = 200 + + SHELL_NAME = 'noop' + + def create_prefix_script(self, prefix_path, merge_install): + return [] + + def create_package_script(self, prefix_path, pkg_name, hooks): + return [] + + def create_hook_set_value( + self, env_hook_name, prefix_path, pkg_name, name, value, + ): + hook_file = \ + prefix_path / 'share' / pkg_name / 'hook' / f'{env_hook_name}.noop' + hook_file.parent.mkdir(parents=True, exist_ok=True) + hook_file.write_text('') + return hook_file + + def create_hook_append_value( + self, env_hook_name, prefix_path, pkg_name, name, subdirectory, + ): + hook_file = \ + prefix_path / 'share' / pkg_name / 'hook' / f'{env_hook_name}.noop' + hook_file.parent.mkdir(parents=True, exist_ok=True) + hook_file.write_text('') + return hook_file + + def create_hook_prepend_value( + self, env_hook_name, prefix_path, pkg_name, name, subdirectory, + ): + hook_file = \ + prefix_path / 'share' / pkg_name / 'hook' / f'{env_hook_name}.noop' + hook_file.parent.mkdir(parents=True, exist_ok=True) + hook_file.write_text('') + return hook_file + + def create_hook_include_file( + self, env_hook_name, prefix_path, pkg_name, relative_path, + ): + hook_file = \ + prefix_path / 'share' / pkg_name / 'hook' / f'{env_hook_name}.noop' + hook_file.parent.mkdir(parents=True, exist_ok=True) + hook_file.write_text('') + return hook_file + + async def generate_command_environment( + self, task_name, build_base, dependencies, + ): + return dict(os.environ) + + +@pytest.fixture(autouse=True, scope='module') +def suppress_shell_extensions(): + with patch( + 'colcon_core.shell.get_shell_extensions', + return_value={ + _NoopShellExtension.PRIORITY: { + _NoopShellExtension.SHELL_NAME: _NoopShellExtension() + }, + }, + ): + yield + + +@pytest.mark.benchmark(group='pipeline.package_identification') +@pytest.mark.parametrize( + 'backend,identifier,_,__', + PIPELINES.values(), + ids=PIPELINES.keys()) +def test_identify(backend, identifier, _, __, request, benchmark): + mock_project = request.getfixturevalue(f'mock_{backend}_project') + + def dut(): + mock_desc = PackageDescriptor(mock_project.path) + assert identifier.identify(mock_desc) is None + return mock_desc + mock_desc = benchmark(dut) + assert mock_desc.identifies_package() + + +@pytest.mark.benchmark(group='pipeline.package_augmentation') +@pytest.mark.parametrize( + 'backend,identifier,augmentor,_', + PIPELINES.values(), + ids=PIPELINES.keys()) +def test_augment(backend, identifier, augmentor, _, request, benchmark): + mock_project = request.getfixturevalue(f'mock_{backend}_project') + mock_project = PackageDescriptor(mock_project.path) + assert identifier.identify(mock_project) is None + assert mock_project.identifies_package() + + @benchmark + def dut(): + mock_desc = deepcopy(mock_project) + assert augmentor.augment_package(mock_desc) is None + + +@pytest.mark.benchmark(group='pipeline.build') +@pytest.mark.parametrize( + 'backend,identifier,augmentor,build_task', + PIPELINES.values(), + ids=PIPELINES.keys()) +def test_build( + backend, identifier, augmentor, build_task, request, bench, + tmp_path_factory +): + mock_project = request.getfixturevalue(f'mock_{backend}_project') + mock_project = PackageDescriptor(mock_project.path) + assert identifier.identify(mock_project) is None + assert mock_project.identifies_package() + assert augmentor.augment_package(mock_project) is None + + async def dut(): + build_task.set_context(context=_NoEventsTaskContext( + pkg=mock_project, + args=SimpleNamespace( + path=str(mock_project.path), + build_base=str(tmp_path_factory.mktemp('build')), + install_base=str(tmp_path_factory.mktemp('install')), + symlink_install=False, + ), + dependencies={}, + )) + + return await build_task.build() + + assert not bench(dut) diff --git a/test/test_spell_check.py b/test/test_spell_check.py index 8032543..60bc823 100644 --- a/test/test_spell_check.py +++ b/test/test_spell_check.py @@ -4,25 +4,24 @@ from pathlib import Path import pytest -from scspell import Report -from scspell import SCSPELL_BUILTIN_DICT -from scspell import spell_check - spell_check_words_path = Path(__file__).parent / 'spell_check.words' @pytest.fixture(scope='module') def known_words(): - global spell_check_words_path return spell_check_words_path.read_text().splitlines() +@pytest.mark.linter def test_spell_check(known_words): + from scspell import Report + from scspell import SCSPELL_BUILTIN_DICT + from scspell import spell_check + source_filenames = [Path(__file__).parents[1] / 'setup.py'] + \ list( - (Path(__file__).parents[1] / - 'colcon_python_project') + (Path(__file__).parents[1] / 'colcon_python_project') .glob('**/*.py')) + \ list((Path(__file__).parents[1] / 'test').glob('**/*.py')) @@ -37,21 +36,23 @@ def test_spell_check(known_words): unknown_word_count = len(report.unknown_words) assert unknown_word_count == 0, \ - 'Found {unknown_word_count} unknown words: '.format_map(locals()) + \ + f'Found {unknown_word_count} unknown words: ' + \ ', '.join(sorted(report.unknown_words)) unused_known_words = set(known_words) - report.found_known_words unused_known_word_count = len(unused_known_words) assert unused_known_word_count == 0, \ - '{unused_known_word_count} words in the word list are not used: ' \ - .format_map(locals()) + ', '.join(sorted(unused_known_words)) + f'{unused_known_word_count} words in the word list are not used: ' + \ + ', '.join(sorted(unused_known_words)) +@pytest.mark.linter def test_spell_check_word_list_order(known_words): assert known_words == sorted(known_words), \ 'The word list should be ordered alphabetically' +@pytest.mark.linter def test_spell_check_word_list_duplicates(known_words): assert len(known_words) == len(set(known_words)), \ 'The word list should not contain duplicates'