From 727508e66da5e4a79fa0a7017fae919365e4c8c5 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Sun, 28 Aug 2022 19:27:21 +1000 Subject: [PATCH 1/8] Standardise on pip_parse --- .bazelci/presubmit.yml | 1 - README.md | 78 +++----- docs/pip.md | 75 +------- docs/pip_repository.md | 22 +-- examples/BUILD | 5 - examples/pip_install/BUILD | 6 +- examples/pip_install/WORKSPACE | 5 + examples/pip_install/pip_install_test.py | 24 +-- examples/pip_install/requirements.txt | 8 +- examples/pip_install/requirements_windows.txt | 6 +- examples/pip_parse/requirements_lock.txt | 6 +- examples/pip_parse_vendored/requirements.bzl | 2 +- examples/pip_parse_vendored/requirements.txt | 2 +- examples/pip_repository_annotations/BUILD | 2 +- examples/pip_repository_annotations/WORKSPACE | 8 +- .../requirements.txt | 2 +- examples/relative_requirements/BUILD | 10 - examples/relative_requirements/README.md | 4 - examples/relative_requirements/WORKSPACE | 21 -- examples/relative_requirements/main.py | 5 - .../relative_package_name/__init__.py | 2 - .../relative_package/setup.py | 7 - .../relative_requirements/requirements.txt | 1 - python/pip.bzl | 94 ++------- python/pip_install/extract_wheels/BUILD | 9 - .../pip_install/extract_wheels/arguments.py | 6 - .../extract_wheels/extract_single_wheel.py | 29 ++- .../extract_wheels/extract_wheels.py | 132 ------------- python/pip_install/pip_compile.py | 17 +- python/pip_install/pip_repository.bzl | 181 +++++++++--------- python/pip_install/private/srcs.bzl | 1 - python/repositories.bzl | 26 ++- tests/BUILD | 5 - tests/pip_deps/BUILD | 19 -- tests/pip_deps/README.md | 1 - tests/pip_deps/WORKSPACE | 26 --- tests/pip_deps/pip_deps.bzl | 55 ------ tests/pip_deps/requirements.txt | 136 ------------- tests/pip_deps/test.sh | 4 - tests/pip_repository_entry_points/WORKSPACE | 8 +- .../requirements.txt | 6 +- 41 files changed, 247 insertions(+), 810 deletions(-) delete mode 100644 examples/relative_requirements/BUILD delete mode 100644 examples/relative_requirements/README.md delete mode 100644 examples/relative_requirements/WORKSPACE delete mode 100644 examples/relative_requirements/main.py delete mode 100644 examples/relative_requirements/relative_package/relative_package_name/__init__.py delete mode 100644 examples/relative_requirements/relative_package/setup.py delete mode 100644 examples/relative_requirements/requirements.txt delete mode 100644 python/pip_install/extract_wheels/extract_wheels.py delete mode 100644 tests/pip_deps/BUILD delete mode 100644 tests/pip_deps/README.md delete mode 100644 tests/pip_deps/WORKSPACE delete mode 100644 tests/pip_deps/pip_deps.bzl delete mode 100644 tests/pip_deps/requirements.txt delete mode 100755 tests/pip_deps/test.sh diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 03687d442c..1cc121a4e7 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -36,6 +36,5 @@ platforms: - "-//gazelle/..." # The dependencies needed for this test are not cross-platform: https://github.com/bazelbuild/rules_python/issues/260 - "-//tests:pip_repository_entry_points_example" - - "-//tests:pip_deps_example" test_flags: - "--test_tag_filters=-fix-windows" \ No newline at end of file diff --git a/README.md b/README.md index 7359a2ae4e..e0335241d3 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@ This repository is the home of the core Python rules -- `py_library`, `py_binary`, `py_test`, and related symbols that provide the basis for Python -support in Bazel. It also contains packaging rules for integrating with PyPI -(`pip`). Documentation lives in the +support in Bazel. It also contains package installation rules for integrating with PyPI and other package indices. Documentation lives in the [`docs/`](https://github.com/bazelbuild/rules_python/tree/main/docs) directory and in the [Bazel Build Encyclopedia](https://docs.bazel.build/versions/master/be/python.html). @@ -24,7 +23,7 @@ Once they are fully migrated to rules_python, they may evolve at a different rate, but this repository will still follow [semantic versioning](https://semver.org). -The packaging rules (`pip_install`, etc.) are less stable. We may make breaking +The package installation rules (`pip_install`, `pip_parse` etc.) are less stable. We may make breaking changes as they evolve. This repository is maintained by the Bazel community. Neither Google, nor the @@ -101,14 +100,14 @@ py_binary( ) ``` -## Using the packaging rules +## Using the package installation rules Usage of the packaging rules involves two main steps. -1. [Installing `pip` dependencies](#installing-pip-dependencies) -2. [Consuming `pip` dependencies](#consuming-pip-dependencies) +1. [Installing third_party packages](#installing-third_party-packages) +2. [Using third_party packages as dependencies](#using-third_party-packages-as-dependencies) -The packaging rules create two kinds of repositories: A central external repo that holds +The package installation rules create two kinds of repositories: A central external repo that holds downloaded wheel files, and individual external repos for each wheel's extracted contents. Users only need to interact with the central external repo; the wheel repos are essentially an implementation detail. The central external repo provides a @@ -116,55 +115,12 @@ are essentially an implementation detail. The central external repo provides a `BUILD` files that translates a pip package name into the label of a `py_library` target in the appropriate wheel repo. -### Installing `pip` dependencies +### Installing third_party packages To add pip dependencies to your `WORKSPACE`, load the `pip_install` function, and call it to create the central external repo and individual wheel external repos. -```python -load("@rules_python//python:pip.bzl", "pip_install") - -# Create a central external repo, @my_deps, that contains Bazel targets for all the -# third-party packages specified in the requirements.txt file. -pip_install( - name = "my_deps", - requirements = "//path/to:requirements.txt", -) -``` - -Note that since `pip_install` is a repository rule and therefore executes pip at WORKSPACE-evaluation time, Bazel has no -information about the Python toolchain and cannot enforce that the interpreter -used to invoke pip matches the interpreter used to run `py_binary` targets. By -default, `pip_install` uses the system command `"python3"`. This can be overridden by passing the -`python_interpreter` attribute or `python_interpreter_target` attribute to `pip_install`. - -You can have multiple `pip_install`s in the same workspace. This will create multiple external repos that have no relation to -one another, and may result in downloading the same wheels multiple times. - -As with any repository rule, if you would like to ensure that `pip_install` is -re-executed in order to pick up a non-hermetic change to your environment (e.g., -updating your system `python` interpreter), you can force it to re-execute by running -`bazel sync --only [pip_install name]`. - -### Fetch `pip` dependencies lazily - -One pain point with `pip_install` is the need to download all dependencies resolved by -your requirements.txt before the bazel analysis phase can start. For large python monorepos -this can take a long time, especially on slow connections. - -`pip_parse` provides a solution to this problem. If you can provide a lock -file of all your python dependencies `pip_parse` will translate each requirement into its own external repository. -Bazel will only fetch/build wheels for the requirements in the subgraph of your build target. - -There are API differences between `pip_parse` and `pip_install`: -1. `pip_parse` requires a fully resolved lock file of your python dependencies. You can generate this by using the `compile_pip_requirements` rule, - running `pip-compile` directly, or using virtualenv and `pip freeze`. `pip_parse` uses a label argument called `requirements_lock` instead of - `requirements` to make this distinction clear. -2. `pip_parse` translates your requirements into a starlark macro called `install_deps`. You must call this macro in your WORKSPACE to - declare your dependencies. - - ```python load("@rules_python//python:pip.bzl", "pip_parse") @@ -174,14 +130,30 @@ pip_parse( name = "my_deps", requirements_lock = "//path/to:requirements_lock.txt", ) - # Load the starlark macro which will define your dependencies. load("@my_deps//:requirements.bzl", "install_deps") # Call it to define repos for your requirements. install_deps() ``` -### Consuming `pip` dependencies +Note that since `pip_parse` is a repository rule and therefore executes pip at WORKSPACE-evaluation time, Bazel has no +information about the Python toolchain and cannot enforce that the interpreter +used to invoke pip matches the interpreter used to run `py_binary` targets. By +default, `pip_parse` uses the system command `"python3"`. This can be overridden by passing the +`python_interpreter` attribute or `python_interpreter_target` attribute to `pip_parse`. + +You can have multiple `pip_parse`s in the same workspace. This will create multiple external repos that have no relation to +one another, and may result in downloading the same wheels multiple times. + +As with any repository rule, if you would like to ensure that `pip_parse` is +re-executed in order to pick up a non-hermetic change to your environment (e.g., +updating your system `python` interpreter), you can force it to re-execute by running +`bazel sync --only [pip_parse name]`. + +Note: The `pip_install` rule is deprecated. `pip_parse` offers identical functionality and both `pip_install` +and `pip_parse` now have the same implementation. The name `pip_install` may be removed in a future version of the rules. + +### Using third_party packages as dependencies Each extracted wheel repo contains a `py_library` target representing the wheel's contents. There are two ways to access this library. The diff --git a/docs/pip.md b/docs/pip.md index 4853e5252d..b302ccfbeb 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -73,70 +73,6 @@ Annotations to apply to the BUILD file content from package generated from a `pi pip_install(requirements, name, kwargs) -Accepts a `requirements.txt` file and installs the dependencies listed within. - -Those dependencies become available in a generated `requirements.bzl` file. - -This macro wraps the [`pip_repository`](./pip_repository.md) rule that invokes `pip`. -In your WORKSPACE file: - -```python -pip_install( - requirements = ":requirements.txt", -) -``` - -You can then reference installed dependencies from a `BUILD` file with: - -```python -load("@pip//:requirements.bzl", "requirement") -py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - requirement("requests"), - requirement("numpy"), - ], -) -``` - -> Note that this convenience comes with a cost. -> Analysis of any BUILD file which loads the requirements helper in this way will -> cause an eager-fetch of all the pip dependencies, -> even if no python targets are requested to be built. -> In a multi-language repo, this may cause developers to fetch dependencies they don't need, -> so consider using the long form for dependencies if this happens. - -In addition to the `requirement` macro, which is used to access the `py_library` -target generated from a package's wheel, the generated `requirements.bzl` file contains -functionality for exposing [entry points][whl_ep] as `py_binary` targets. - -[whl_ep]: https://packaging.python.org/specifications/entry-points/ - -```python -load("@pip_deps//:requirements.bzl", "entry_point") - -alias( - name = "pip-compile", - actual = entry_point( - pkg = "pip-tools", - script = "pip-compile", - ), -) -``` - -Note that for packages whose name and script are the same, only the name of the package -is needed when calling the `entry_point` macro. - -```python -load("@pip_deps//:requirements.bzl", "entry_point") - -alias( - name = "flake8", - actual = entry_point("flake8"), -) -``` **PARAMETERS** @@ -144,9 +80,9 @@ alias( | Name | Description | Default Value | | :-------------: | :-------------: | :-------------: | -| requirements | A 'requirements.txt' pip requirements file. | None | -| name | A unique name for the created external repository (default 'pip'). | "pip" | -| kwargs | Additional arguments to the [pip_repository](./pip_repository.md) repository rule. | none | +| requirements |

-

| None | +| name |

-

| "pip" | +| kwargs |

-

| none | @@ -154,7 +90,7 @@ alias( ## pip_parse 
-pip_parse(requirements_lock, name, kwargs)
+pip_parse(requirements, requirements_lock, name, kwargs)
Accepts a locked/compiled requirements file and installs the dependencies listed within. @@ -247,7 +183,8 @@ See the example in rules_python/examples/pip_parse_vendored. | Name | Description | Default Value | | :-------------: | :-------------: | :-------------: | -| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. Note that if your lockfile is platform-dependent, you can use the requirements_[platform] attributes. | none | +| requirements | Deprecated. See requirements_lock. | None | +| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. Note that if your lockfile is platform-dependent, you can use the requirements_[platform] attributes. | None | | name | The name of the generated repository. The generated repositories containing each requirement will be of the form <name>_<requirement-name>. | "pip_parsed_deps" | | kwargs | Additional arguments to the [pip_repository](./pip_repository.md) repository rule. | none | diff --git a/docs/pip_repository.md b/docs/pip_repository.md index c66d8bfd91..9294cf1a5b 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -5,10 +5,10 @@ ## pip_repository 
-pip_repository(name, annotations, download_only, enable_implicit_namespace_pkgs, environment,
-               extra_pip_args, incremental, isolated, pip_data_exclude, python_interpreter,
-               python_interpreter_target, quiet, repo_prefix, requirements, requirements_darwin,
-               requirements_linux, requirements_lock, requirements_windows, timeout)
+pip_repository(name, annotations, enable_implicit_namespace_pkgs, environment, extra_pip_args,
+               isolated, pip_data_exclude, python_interpreter, python_interpreter_target, quiet,
+               repo_prefix, requirements_darwin, requirements_linux, requirements_lock,
+               requirements_windows, timeout)
A rule for importing `requirements.txt` dependencies into Bazel. @@ -58,18 +58,15 @@ py_binary( | :-------------: | :-------------: | :-------------: | :-------------: | :-------------: | | name | A unique name for this repository. | Name | required | | | annotations | Optional annotations to apply to packages | Dictionary: String -> String | optional | {} | -| download_only | Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of --platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different platform from the host platform. | Boolean | optional | False | | enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False | | environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | Dictionary: String -> String | optional | {} | | extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] | -| incremental | Create the repository in incremental mode. | Boolean | optional | False | | isolated | Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to the underlying pip command. Alternatively, the RULES_PYTHON_PIP_ISOLATED enviornment varaible can be used to control this flag. | Boolean | optional | True | | pip_data_exclude | Additional data exclusion parameters to add to the pip packages BUILD file. | List of strings | optional | [] | | python_interpreter | The python interpreter to use. This can either be an absolute path or the name of a binary found on the host's PATH environment variable. If no value is set python3 is defaulted for Unix systems and python.exe for Windows. | String | optional | "" | | python_interpreter_target | If you are using a custom python interpreter built by another repository rule, use this attribute to specify its BUILD target. This allows pip_repository to invoke pip using the same interpreter as your toolchain. If set, takes precedence over python_interpreter. | Label | optional | None | | quiet | If True, suppress printing stdout and stderr output to the terminal. | Boolean | optional | True | -| repo_prefix | Prefix for the generated packages. For non-incremental mode the packages will be of the form

@<name>//<prefix><sanitized-package-name>/...

For incremental mode the packages will be of the form

@<prefix><sanitized-package-name>//... | String | optional | "" | -| requirements | A 'requirements.txt' pip requirements file. | Label | optional | None | +| repo_prefix | Prefix for the generated packages will be of the form

@<prefix><sanitized-package-name>//... | String | optional | "" | | requirements_darwin | Override the requirements_lock attribute when the host platform is Mac OS | Label | optional | None | | requirements_linux | Override the requirements_lock attribute when the host platform is Linux | Label | optional | None | | requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. | Label | optional | None | @@ -82,9 +79,9 @@ py_binary( ## whl_library 
-whl_library(name, annotation, download_only, enable_implicit_namespace_pkgs, environment,
-            extra_pip_args, isolated, pip_data_exclude, python_interpreter, python_interpreter_target,
-            quiet, repo, repo_prefix, requirement, timeout)
+whl_library(name, annotation, enable_implicit_namespace_pkgs, environment, extra_pip_args, isolated,
+            pip_data_exclude, python_interpreter, python_interpreter_target, quiet, repo, repo_prefix,
+            requirement, timeout)
@@ -98,7 +95,6 @@ Instantiated from pip_repository and inherits config options from there. | :-------------: | :-------------: | :-------------: | :-------------: | :-------------: | | name | A unique name for this repository. | Name | required | | | annotation | Optional json encoded file containing annotation to apply to the extracted wheel. See package_annotation | Label | optional | None | -| download_only | Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of --platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different platform from the host platform. | Boolean | optional | False | | enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False | | environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | Dictionary: String -> String | optional | {} | | extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] | @@ -108,7 +104,7 @@ Instantiated from pip_repository and inherits config options from there. | python_interpreter_target | If you are using a custom python interpreter built by another repository rule, use this attribute to specify its BUILD target. This allows pip_repository to invoke pip using the same interpreter as your toolchain. If set, takes precedence over python_interpreter. | Label | optional | None | | quiet | If True, suppress printing stdout and stderr output to the terminal. | Boolean | optional | True | | repo | Pointer to parent repo name. Used to make these rules rerun if the parent repo changes. | String | required | | -| repo_prefix | Prefix for the generated packages. For non-incremental mode the packages will be of the form

@<name>//<prefix><sanitized-package-name>/...

For incremental mode the packages will be of the form

@<prefix><sanitized-package-name>//... | String | optional | "" | +| repo_prefix | Prefix for the generated packages will be of the form

@<prefix><sanitized-package-name>//... | String | optional | "" | | requirement | Python requirement string describing the package to make available | String | required | | | timeout | Timeout (in seconds) on the rule's execution duration. | Integer | optional | 600 | diff --git a/examples/BUILD b/examples/BUILD index 41dd87505a..ee4d7e4861 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -38,11 +38,6 @@ bazel_integration_test( timeout = "long", ) -bazel_integration_test( - name = "relative_requirements_example", - timeout = "long", -) - bazel_integration_test( name = "bzlmod_example", ) diff --git a/examples/pip_install/BUILD b/examples/pip_install/BUILD index ad983b2f54..35f5a9338a 100644 --- a/examples/pip_install/BUILD +++ b/examples/pip_install/BUILD @@ -88,9 +88,9 @@ py_test( genquery( name = "yamllint_lib_by_version", expression = """ - attr("tags", "\\bpypi_version=1.26.3\\b", "@pip//pypi__yamllint") + attr("tags", "\\bpypi_version=1.26.3\\b", "@pip_yamllint//:pkg") intersect - attr("tags", "\\bpypi_name=yamllint\\b", "@pip//pypi__yamllint") + attr("tags", "\\bpypi_name=yamllint\\b", "@pip_yamllint//:pkg") """, scope = [requirement("yamllint")], ) @@ -99,7 +99,7 @@ write_file( name = "write_expected", out = "expected", content = [ - "@pip//pypi__yamllint:pypi__yamllint", + "@pip_yamllint//:pkg", "", ], ) diff --git a/examples/pip_install/WORKSPACE b/examples/pip_install/WORKSPACE index 0b33a2b390..f63d928013 100644 --- a/examples/pip_install/WORKSPACE +++ b/examples/pip_install/WORKSPACE @@ -57,6 +57,11 @@ pip_install( requirements = "//:requirements.txt", ) +load("@pip//:requirements.bzl", "install_deps") + +# Initialize repositories for all packages in requirements.txt. +install_deps() + # You could optionally use an in-build, compiled python interpreter as a toolchain, # and also use it to execute pip. # diff --git a/examples/pip_install/pip_install_test.py b/examples/pip_install/pip_install_test.py index 6092768da6..9fe51fa3a4 100644 --- a/examples/pip_install/pip_install_test.py +++ b/examples/pip_install/pip_install_test.py @@ -37,11 +37,11 @@ def test_data(self): self.assertListEqual( env.split(" "), [ - "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/INSTALL.md", - "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/LICENSE", - "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/NEWS", - "external/pip/pypi__s3cmd/data/share/doc/packages/s3cmd/README.md", - "external/pip/pypi__s3cmd/data/share/man/man1/s3cmd.1", + "external/pip_s3cmd/data/share/doc/packages/s3cmd/INSTALL.md", + "external/pip_s3cmd/data/share/doc/packages/s3cmd/LICENSE", + "external/pip_s3cmd/data/share/doc/packages/s3cmd/NEWS", + "external/pip_s3cmd/data/share/doc/packages/s3cmd/README.md", + "external/pip_s3cmd/data/share/man/man1/s3cmd.1", ], ) @@ -51,13 +51,13 @@ def test_dist_info(self): self.assertListEqual( env.split(" "), [ - "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/DESCRIPTION.rst", - "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/INSTALLER", - "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/METADATA", - "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/RECORD", - "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/WHEEL", - "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/metadata.json", - "external/pip/pypi__boto3/site-packages/boto3-1.14.51.dist-info/top_level.txt", + "external/pip_boto3/site-packages/boto3-1.14.51.dist-info/DESCRIPTION.rst", + "external/pip_boto3/site-packages/boto3-1.14.51.dist-info/INSTALLER", + "external/pip_boto3/site-packages/boto3-1.14.51.dist-info/METADATA", + "external/pip_boto3/site-packages/boto3-1.14.51.dist-info/RECORD", + "external/pip_boto3/site-packages/boto3-1.14.51.dist-info/WHEEL", + "external/pip_boto3/site-packages/boto3-1.14.51.dist-info/metadata.json", + "external/pip_boto3/site-packages/boto3-1.14.51.dist-info/top_level.txt", ], ) diff --git a/examples/pip_install/requirements.txt b/examples/pip_install/requirements.txt index db76801f39..8a06da02b6 100644 --- a/examples/pip_install/requirements.txt +++ b/examples/pip_install/requirements.txt @@ -7,7 +7,7 @@ boto3==1.14.51 \ --hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \ --hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71 - # via -r requirements.in + # via -r ./requirements.in botocore==1.17.63 \ --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \ --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75 @@ -77,7 +77,7 @@ pyyaml==6.0 \ s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r requirements.in + # via -r ./requirements.in s3transfer==0.3.7 \ --hash=sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994 \ --hash=sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246 @@ -89,14 +89,14 @@ six==1.16.0 \ tree-sitter==0.20.0 ; sys_platform != "win32" \ --hash=sha256:1940f64be1e8c9c3c0e34a2258f1e4c324207534d5b1eefc5ab2960a9d98f668 \ --hash=sha256:51a609a7c1bd9d9e75d92ee128c12c7852ae70a482900fbbccf3d13a79e0378c - # via -r requirements.in + # via -r ./requirements.in urllib3==1.25.11 \ --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e # via botocore yamllint==1.26.3 \ --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e - # via -r requirements.in + # via -r ./requirements.in # The following packages are considered to be unsafe in a requirements file: setuptools==59.6.0 \ diff --git a/examples/pip_install/requirements_windows.txt b/examples/pip_install/requirements_windows.txt index 26de1adaac..09a6a83c5d 100644 --- a/examples/pip_install/requirements_windows.txt +++ b/examples/pip_install/requirements_windows.txt @@ -7,7 +7,7 @@ boto3==1.14.51 \ --hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \ --hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71 - # via -r requirements.in + # via -r ./requirements.in botocore==1.17.63 \ --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \ --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75 @@ -77,7 +77,7 @@ pyyaml==6.0 \ s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r requirements.in + # via -r ./requirements.in s3transfer==0.3.7 \ --hash=sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994 \ --hash=sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246 @@ -92,7 +92,7 @@ urllib3==1.25.11 \ # via botocore yamllint==1.26.3 \ --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e - # via -r requirements.in + # via -r ./requirements.in # The following packages are considered to be unsafe in a requirements file: setuptools==59.6.0 \ diff --git a/examples/pip_parse/requirements_lock.txt b/examples/pip_parse/requirements_lock.txt index d3cb1f5bc9..a54d912d6a 100644 --- a/examples/pip_parse/requirements_lock.txt +++ b/examples/pip_parse/requirements_lock.txt @@ -66,11 +66,11 @@ pyyaml==6.0 \ requests==2.25.1 \ --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e - # via -r requirements.in + # via -r ./requirements.in s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 - # via -r requirements.in + # via -r ./requirements.in six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -81,7 +81,7 @@ urllib3==1.26.7 \ # via requests yamllint==1.26.3 \ --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e - # via -r requirements.in + # via -r ./requirements.in # The following packages are considered to be unsafe in a requirements file: setuptools==59.6.0 \ diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 58c6e7bad1..038b612309 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -12,7 +12,7 @@ all_requirements = ["@pip_certifi//:pkg", "@pip_charset_normalizer//:pkg", "@pip all_whl_requirements = ["@pip_certifi//:whl", "@pip_charset_normalizer//:whl", "@pip_idna//:whl", "@pip_requests//:whl", "@pip_urllib3//:whl"] _packages = [("pip_certifi", "certifi==2021.10.8 --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"), ("pip_charset_normalizer", "charset-normalizer==2.0.12 --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"), ("pip_idna", "idna==3.3 --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"), ("pip_requests", "requests==2.27.1 --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"), ("pip_urllib3", "urllib3==1.26.9 --hash=sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14 --hash=sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e")] -_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} +_config = {"enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} _annotations = {} def _clean_name(name): diff --git a/examples/pip_parse_vendored/requirements.txt b/examples/pip_parse_vendored/requirements.txt index 81b56154c2..d2dfc20576 100644 --- a/examples/pip_parse_vendored/requirements.txt +++ b/examples/pip_parse_vendored/requirements.txt @@ -19,7 +19,7 @@ idna==3.3 \ requests==2.27.1 \ --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \ --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d - # via -r requirements.in + # via -r ./requirements.in urllib3==1.26.9 \ --hash=sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14 \ --hash=sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e diff --git a/examples/pip_repository_annotations/BUILD b/examples/pip_repository_annotations/BUILD index 8c69c40aff..4fd124e6c7 100644 --- a/examples/pip_repository_annotations/BUILD +++ b/examples/pip_repository_annotations/BUILD @@ -27,7 +27,7 @@ py_test( py_test( name = "pip_install_annotations_test", srcs = ["pip_repository_annotations_test.py"], - env = {"WHEEL_PKG_DIR": "pip_installed/pypi__wheel"}, + env = {"WHEEL_PKG_DIR": "pip_installed_wheel"}, main = "pip_repository_annotations_test.py", deps = [ requirement("wheel"), diff --git a/examples/pip_repository_annotations/WORKSPACE b/examples/pip_repository_annotations/WORKSPACE index 8ee885d468..aeea84207c 100644 --- a/examples/pip_repository_annotations/WORKSPACE +++ b/examples/pip_repository_annotations/WORKSPACE @@ -54,9 +54,9 @@ pip_parse( requirements_lock = "//:requirements.txt", ) -load("@pip_parsed//:requirements.bzl", "install_deps") +load("@pip_parsed//:requirements.bzl", install_pip_parse_deps = "install_deps") -install_deps() +install_pip_parse_deps() # For a more thorough example of `pip_install`. See `@rules_python//examples/pip_install` pip_install( @@ -65,3 +65,7 @@ pip_install( python_interpreter_target = interpreter, requirements = "//:requirements.txt", ) + +load("@pip_installed//:requirements.bzl", install_pip_install_deps = "install_deps") + +install_pip_install_deps() diff --git a/examples/pip_repository_annotations/requirements.txt b/examples/pip_repository_annotations/requirements.txt index e1c53c27e3..a2f161392a 100644 --- a/examples/pip_repository_annotations/requirements.txt +++ b/examples/pip_repository_annotations/requirements.txt @@ -9,4 +9,4 @@ wheel==0.37.1 \ --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 - # via -r requirements.in + # via -r ./requirements.in diff --git a/examples/relative_requirements/BUILD b/examples/relative_requirements/BUILD deleted file mode 100644 index d24ee5f72b..0000000000 --- a/examples/relative_requirements/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -load("@pip//:requirements.bzl", "requirement") -load("@rules_python//python:defs.bzl", "py_test") - -py_test( - name = "main", - srcs = ["main.py"], - deps = [ - requirement("relative_package_name"), - ], -) diff --git a/examples/relative_requirements/README.md b/examples/relative_requirements/README.md deleted file mode 100644 index 4b9258e370..0000000000 --- a/examples/relative_requirements/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# relative_requirements example - -This example shows how to use pip to fetch relative dependencies from a requirements.txt file, -then use them in BUILD files as dependencies of Bazel targets. diff --git a/examples/relative_requirements/WORKSPACE b/examples/relative_requirements/WORKSPACE deleted file mode 100644 index 4ae91c39d8..0000000000 --- a/examples/relative_requirements/WORKSPACE +++ /dev/null @@ -1,21 +0,0 @@ -workspace(name = "example_repo") - -local_repository( - name = "rules_python", - path = "../..", -) - -load("@rules_python//python:repositories.bzl", "python_register_toolchains") - -python_register_toolchains( - name = "python39", - python_version = "3.9", -) - -load("@python39//:defs.bzl", "interpreter") -load("@rules_python//python:pip.bzl", "pip_install") - -pip_install( - python_interpreter_target = interpreter, - requirements = "//:requirements.txt", -) diff --git a/examples/relative_requirements/main.py b/examples/relative_requirements/main.py deleted file mode 100644 index b8ac021e90..0000000000 --- a/examples/relative_requirements/main.py +++ /dev/null @@ -1,5 +0,0 @@ -import relative_package_name - -if __name__ == "__main__": - # Run a function from the relative package - print(relative_package_name.test()) diff --git a/examples/relative_requirements/relative_package/relative_package_name/__init__.py b/examples/relative_requirements/relative_package/relative_package_name/__init__.py deleted file mode 100644 index c031192907..0000000000 --- a/examples/relative_requirements/relative_package/relative_package_name/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -def test(): - return True diff --git a/examples/relative_requirements/relative_package/setup.py b/examples/relative_requirements/relative_package/setup.py deleted file mode 100644 index 052b519345..0000000000 --- a/examples/relative_requirements/relative_package/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -from setuptools import setup - -setup( - name="relative_package_name", - version="1.0.0", - packages=["relative_package_name"], -) diff --git a/examples/relative_requirements/requirements.txt b/examples/relative_requirements/requirements.txt deleted file mode 100644 index 9a81317e1e..0000000000 --- a/examples/relative_requirements/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -./relative_package diff --git a/python/pip.bzl b/python/pip.bzl index 954317f4b7..2419660b5f 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -21,88 +21,11 @@ compile_pip_requirements = _compile_pip_requirements package_annotation = _package_annotation def pip_install(requirements = None, name = "pip", **kwargs): - """Accepts a `requirements.txt` file and installs the dependencies listed within. + # pip_install is now considered deprecated. + # In future, this may log a warning and eventually be removed. + pip_parse(requirements = requirements, name = name, **kwargs) - Those dependencies become available in a generated `requirements.bzl` file. - - This macro wraps the [`pip_repository`](./pip_repository.md) rule that invokes `pip`. - In your WORKSPACE file: - - ```python - pip_install( - requirements = ":requirements.txt", - ) - ``` - - You can then reference installed dependencies from a `BUILD` file with: - - ```python - load("@pip//:requirements.bzl", "requirement") - py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - requirement("requests"), - requirement("numpy"), - ], - ) - ``` - - > Note that this convenience comes with a cost. - > Analysis of any BUILD file which loads the requirements helper in this way will - > cause an eager-fetch of all the pip dependencies, - > even if no python targets are requested to be built. - > In a multi-language repo, this may cause developers to fetch dependencies they don't need, - > so consider using the long form for dependencies if this happens. - - In addition to the `requirement` macro, which is used to access the `py_library` - target generated from a package's wheel, the generated `requirements.bzl` file contains - functionality for exposing [entry points][whl_ep] as `py_binary` targets. - - [whl_ep]: https://packaging.python.org/specifications/entry-points/ - - ```python - load("@pip_deps//:requirements.bzl", "entry_point") - - alias( - name = "pip-compile", - actual = entry_point( - pkg = "pip-tools", - script = "pip-compile", - ), - ) - ``` - - Note that for packages whose name and script are the same, only the name of the package - is needed when calling the `entry_point` macro. - - ```python - load("@pip_deps//:requirements.bzl", "entry_point") - - alias( - name = "flake8", - actual = entry_point("flake8"), - ) - ``` - - Args: - requirements (Label): A 'requirements.txt' pip requirements file. - name (str, optional): A unique name for the created external repository (default 'pip'). - **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule. - """ - - # Just in case our dependencies weren't already fetched - pip_install_dependencies() - - pip_repository( - name = name, - requirements = requirements, - repo_prefix = "pypi__", - **kwargs - ) - -def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): +def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_deps", **kwargs): """Accepts a locked/compiled requirements file and installs the dependencies listed within. Those dependencies become available in a generated `requirements.bzl` file. @@ -195,6 +118,7 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): fetched/built only for the targets specified by 'build/run/test'. Note that if your lockfile is platform-dependent, you can use the `requirements_[platform]` attributes. + requirements (Label): Deprecated. See requirements_lock. name (str, optional): The name of the generated repository. The generated repositories containing each requirement will be of the form _. **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule. @@ -203,10 +127,14 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs): # Just in case our dependencies weren't already fetched pip_install_dependencies() + # Temporary compatibility shim + # pip_install was previously document to use requirements while pip_parse was using requirements_lock + # We would prefer everyone move to using requirements_lock, but we maintain a temporary shim + reqs_to_use = requirements_lock if requirements_lock else requirements + pip_repository( name = name, - requirements_lock = requirements_lock, + requirements_lock = reqs_to_use, repo_prefix = "{}_".format(name), - incremental = True, **kwargs ) diff --git a/python/pip_install/extract_wheels/BUILD b/python/pip_install/extract_wheels/BUILD index 158d34ba27..bc11885026 100644 --- a/python/pip_install/extract_wheels/BUILD +++ b/python/pip_install/extract_wheels/BUILD @@ -9,7 +9,6 @@ py_library( "arguments.py", "bazel.py", "extract_single_wheel.py", - "extract_wheels.py", "namespace_pkgs.py", "parse_requirements_to_bzl.py", "requirements.py", @@ -21,14 +20,6 @@ py_library( ], ) -py_binary( - name = "extract_wheels", - srcs = [ - "extract_wheels.py", - ], - deps = [":lib"], -) - py_binary( name = "extract_single_wheel", srcs = [ diff --git a/python/pip_install/extract_wheels/arguments.py b/python/pip_install/extract_wheels/arguments.py index ce77bb028e..d7d34523a7 100644 --- a/python/pip_install/extract_wheels/arguments.py +++ b/python/pip_install/extract_wheels/arguments.py @@ -39,12 +39,6 @@ def parse_common_args(parser: ArgumentParser) -> ArgumentParser: required=True, help="Prefix to prepend to packages", ) - parser.add_argument( - "--download_only", - action="store_true", - help="Use 'pip download' instead of 'pip wheel'. Disables building wheels from source, but allows use of " - "--platform, --python-version, --implementation, and --abi in --extra_pip_args.", - ) return parser diff --git a/python/pip_install/extract_wheels/extract_single_wheel.py b/python/pip_install/extract_wheels/extract_single_wheel.py index a7cc672a76..1160dbf98d 100644 --- a/python/pip_install/extract_wheels/extract_single_wheel.py +++ b/python/pip_install/extract_wheels/extract_single_wheel.py @@ -8,11 +8,32 @@ from python.pip_install.extract_wheels import arguments, bazel, requirements from python.pip_install.extract_wheels.annotation import annotation_from_str_path -from python.pip_install.extract_wheels.extract_wheels import ( - configure_reproducible_wheels, -) +def configure_reproducible_wheels() -> None: + """Modifies the environment to make wheel building reproducible. + Wheels created from sdists are not reproducible by default. We can however workaround this by + patching in some configuration with environment variables. + """ + + # wheel, by default, enables debug symbols in GCC. This incidentally captures the build path in the .so file + # We can override this behavior by disabling debug symbols entirely. + # https://github.com/pypa/pip/issues/6505 + if "CFLAGS" in os.environ: + os.environ["CFLAGS"] += " -g0" + else: + os.environ["CFLAGS"] = "-g0" + + # set SOURCE_DATE_EPOCH to 1980 so that we can use python wheels + # https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl + if "SOURCE_DATE_EPOCH" not in os.environ: + os.environ["SOURCE_DATE_EPOCH"] = "315532800" + + # Python wheel metadata files can be unstable. + # See https://bitbucket.org/pypa/wheel/pull-requests/74/make-the-output-of-metadata-files/diff + if "PYTHONHASHSEED" not in os.environ: + os.environ["PYTHONHASHSEED"] = "0" + def main() -> None: parser = argparse.ArgumentParser( description="Build and/or fetch a single wheel based on the requirement passed in" @@ -38,7 +59,7 @@ def main() -> None: pip_args = ( [sys.executable, "-m", "pip"] + (["--isolated"] if args.isolated else []) - + ["download" if args.download_only else "wheel", "--no-deps"] + + ["wheel", "--no-deps"] + deserialized_args["extra_pip_args"] ) diff --git a/python/pip_install/extract_wheels/extract_wheels.py b/python/pip_install/extract_wheels/extract_wheels.py deleted file mode 100644 index 2addaf89fd..0000000000 --- a/python/pip_install/extract_wheels/extract_wheels.py +++ /dev/null @@ -1,132 +0,0 @@ -"""extract_wheels - -extract_wheels resolves and fetches artifacts transitively from the Python Package Index (PyPI) based on a -requirements.txt. It generates the required BUILD files to consume these packages as Python libraries. - -Under the hood, it depends on the `pip wheel` command to do resolution, download, and compilation into wheels. -""" -import argparse -import glob -import os -import pathlib -import subprocess -import sys - -from python.pip_install.extract_wheels import ( - annotation, - arguments, - bazel, - requirements, - wheel, -) - - -def configure_reproducible_wheels() -> None: - """Modifies the environment to make wheel building reproducible. - - Wheels created from sdists are not reproducible by default. We can however workaround this by - patching in some configuration with environment variables. - """ - - # wheel, by default, enables debug symbols in GCC. This incidentally captures the build path in the .so file - # We can override this behavior by disabling debug symbols entirely. - # https://github.com/pypa/pip/issues/6505 - if "CFLAGS" in os.environ: - os.environ["CFLAGS"] += " -g0" - else: - os.environ["CFLAGS"] = "-g0" - - # set SOURCE_DATE_EPOCH to 1980 so that we can use python wheels - # https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl - if "SOURCE_DATE_EPOCH" not in os.environ: - os.environ["SOURCE_DATE_EPOCH"] = "315532800" - - # Python wheel metadata files can be unstable. - # See https://bitbucket.org/pypa/wheel/pull-requests/74/make-the-output-of-metadata-files/diff - if "PYTHONHASHSEED" not in os.environ: - os.environ["PYTHONHASHSEED"] = "0" - - -def main() -> None: - """Main program. - - Exits zero on successful program termination, non-zero otherwise. - """ - - configure_reproducible_wheels() - - parser = argparse.ArgumentParser( - description="Resolve and fetch artifacts transitively from PyPI" - ) - parser.add_argument( - "--requirements", - action="store", - required=True, - help="Path to requirements.txt from where to install dependencies", - ) - parser.add_argument( - "--annotations", - type=annotation.annotations_map_from_str_path, - help="A json encoded file containing annotations for rendered packages.", - ) - arguments.parse_common_args(parser) - args = parser.parse_args() - deserialized_args = dict(vars(args)) - arguments.deserialize_structured_args(deserialized_args) - - # Pip is run with the working directory changed to the folder containing the requirements.txt file, to allow for - # relative requirements to be correctly resolved. The --wheel-dir is therefore required to be repointed back to the - # current calling working directory (the repo root in .../external/name), where the wheel files should be written to - pip_args = ( - [sys.executable, "-m", "pip"] - + (["--isolated"] if args.isolated else []) - + ["download" if args.download_only else "wheel", "-r", args.requirements] - + ["--wheel-dir", os.getcwd()] - + deserialized_args["extra_pip_args"] - ) - - env = os.environ.copy() - env.update(deserialized_args["environment"]) - - # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails - subprocess.run( - pip_args, - check=True, - env=env, - cwd=str(pathlib.Path(args.requirements).parent.resolve()), - ) - - extras = requirements.parse_extras(args.requirements) - - repo_label = "@%s" % args.repo - - # Locate all wheels - wheels = [whl for whl in glob.glob("*.whl")] - - # Collect all annotations - reqs = {whl: wheel.Wheel(whl).name for whl in wheels} - annotations = args.annotations.collect(reqs.values()) - - targets = [ - '"{}{}"'.format( - repo_label, - bazel.extract_wheel( - wheel_file=whl, - extras=extras, - pip_data_exclude=deserialized_args["pip_data_exclude"], - enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs, - repo_prefix=args.repo_prefix, - annotation=annotations.get(name), - ), - ) - for whl, name in reqs.items() - ] - - with open("requirements.bzl", "w") as requirement_file: - requirement_file.write( - bazel.generate_requirements_file_contents(repo_label, targets) - ) - - -if __name__ == "__main__": - main() diff --git a/python/pip_install/pip_compile.py b/python/pip_install/pip_compile.py index 09b3c9b035..09f07fefd5 100644 --- a/python/pip_install/pip_compile.py +++ b/python/pip_install/pip_compile.py @@ -35,20 +35,11 @@ def _select_golden_requirements_file( parse_str_none = lambda s: None if s == "None" else s - requirements_in = os.path.relpath(sys.argv.pop(1)) - requirements_txt = os.path.relpath(sys.argv.pop(1)) + requirements_in = sys.argv.pop(1) + requirements_txt = sys.argv.pop(1) requirements_linux = parse_str_none(sys.argv.pop(1)) requirements_darwin = parse_str_none(sys.argv.pop(1)) requirements_windows = parse_str_none(sys.argv.pop(1)) - parts = requirements_in.split(os.path.sep, 2) - if parts[0] == "external": - requirements_in = parts[2] - requirements_txt = ( - requirements_txt - if "BUILD_WORKSPACE_DIRECTORY" in os.environ - else os.path.join("..", "..", requirements_txt) - ) - os.chdir(os.path.join(parts[0], parts[1])) update_target_label = sys.argv.pop(1) # Before loading click, set the locale for its parser. @@ -83,9 +74,7 @@ def _select_golden_requirements_file( # # Changing to the WORKSPACE root avoids 'file not found' errors when the `.update` target is run # from different directories within the WORKSPACE. - requirements_txt = os.path.join( - os.environ["BUILD_WORKSPACE_DIRECTORY"], requirements_txt - ) + os.chdir(os.environ["BUILD_WORKSPACE_DIRECTORY"]) else: err_msg = ( "Expected to find BUILD_WORKSPACE_DIRECTORY (running under `bazel run`) or " diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index eab59f35cd..2f5e9f7e35 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -1,6 +1,6 @@ "" -load("//python:repositories.bzl", "STANDALONE_INTERPRETER_FILENAME") +load("//python:repositories.bzl", "is_standalone_interpreter") load("//python/pip_install:repositories.bzl", "all_requirements") load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") @@ -66,35 +66,68 @@ def _resolve_python_interpreter(rctx): fail("python interpreter `{}` not found in PATH".format(python_interpreter)) return python_interpreter -def _maybe_set_xcode_location_cflags(rctx, environment): - """Patch environment with CPPFLAGS of xcode sdk location. +def _get_xcode_location_cflags(rctx): + """Query the xcode sdk location to update cflags Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so. Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 """ - if ( - rctx.os.name.lower().startswith("mac os") and - rctx.attr.python_interpreter_target != None and - # This is a rules_python provided toolchain. - rctx.execute([ - "ls", - "{}/{}".format( - rctx.path(Label("@{}//:WORKSPACE".format(rctx.attr.python_interpreter_target.workspace_name))).dirname, - STANDALONE_INTERPRETER_FILENAME, - ), - ]).return_code == 0 and - not environment.get(CPPFLAGS) - ): - xcode_sdk_location = rctx.execute(["xcode-select", "--print-path"]) - if xcode_sdk_location.return_code == 0: - xcode_root = xcode_sdk_location.stdout.strip() - if COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): - # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer - # so we need to change the path to to the macos specific tools which are in a different relative - # path than xcode installed command line tools. - xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root) - environment[CPPFLAGS] = "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root) + + # Only run on MacOS hosts + if not rctx.os.name.lower().startswith("mac os"): + return [] + + # Only update the location when using a hermetic toolchain. + if not is_standalone_interpreter(rctx, rctx.attr.python_interpreter_target): + return [] + + # Locate xcode-select + xcode_select = rctx.which("xcode-select") + + xcode_sdk_location = rctx.execute([xcode_select, "--print-path"]) + if xcode_sdk_location.return_code != 0: + return [] + + xcode_root = xcode_sdk_location.stdout.strip() + if COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): + # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer + # so we need to change the path to to the macos specific tools which are in a different relative + # path than xcode installed command line tools. + xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root) + return [ + "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root), + ] + +def _get_toolchain_unix_cflags(rctx): + """Gather cflags from a standalone toolchain for unix systems. + + Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg + otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 + """ + + # Only run on Unix systems + if not rctx.os.name.lower().startswith(("mac os", "linux")): + return [] + + # Only update the location when using a standalone toolchain. + if not is_standalone_interpreter(rctx, rctx.attr.python_interpreter_target): + return [] + + er = rctx.execute([ + rctx.path(rctx.attr.python_interpreter_target).realpath, + "-c", + "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')", + ]) + if er.return_code != 0: + fail("could not get python version from interpreter (status {}): {}".format(er.return_code, er.stderr)) + _python_version = er.stdout + include_path = "{}/include/python{}".format( + rctx.path(Label("@{}//:WORKSPACE".format(rctx.attr.python_interpreter_target.workspace_name))).dirname.realpath, + _python_version, + ) + + return ["-isystem {}".format(include_path)] def _parse_optional_attrs(rctx, args): """Helper function to parse common attributes of pip_repository and whl_library repository rules. @@ -130,9 +163,6 @@ def _parse_optional_attrs(rctx, args): struct(arg = rctx.attr.extra_pip_args).to_json(), ] - if rctx.attr.download_only: - args.append("--download_only") - if rctx.attr.pip_data_exclude != None: args += [ "--pip_data_exclude", @@ -155,10 +185,20 @@ def _create_repository_execution_environment(rctx): Args: rctx: The repository context. - Returns: Dictionary of envrionment variable suitable to pass to rctx.execute. + Returns: + Dictionary of environment variable suitable to pass to rctx.execute. """ - env = {"PYTHONPATH": _construct_pypath(rctx)} - _maybe_set_xcode_location_cflags(rctx, env) + + # Gather any available CPPFLAGS values + cppflags = [] + cppflags.extend(_get_xcode_location_cflags(rctx)) + cppflags.extend(_get_toolchain_unix_cflags(rctx)) + + env = { + "PYTHONPATH": _construct_pypath(rctx), + CPPFLAGS: " ".join(cppflags), + } + return env _BUILD_FILE_CONTENTS = """\ @@ -179,8 +219,7 @@ def _locked_requirements(rctx): requirements_txt = rctx.attr.requirements_windows if not requirements_txt: fail("""\ -Incremental mode requires a requirements_lock attribute be specified, -or a platform-specific lockfile using one of the requirements_* attributes. +A requirements_lock attribute must be specified, or a platform-specific lockfile using one of the requirements_* attributes. """) return requirements_txt @@ -192,40 +231,28 @@ def _pip_repository_impl(rctx): annotations_file = rctx.path("annotations.json") rctx.file(annotations_file, json.encode_indent(annotations, indent = " " * 4)) - if rctx.attr.incremental: - requirements_txt = _locked_requirements(rctx) - args = [ - python_interpreter, - "-m", - "python.pip_install.extract_wheels.parse_requirements_to_bzl", - "--requirements_lock", - rctx.path(requirements_txt), - "--requirements_lock_label", - str(requirements_txt), - # pass quiet and timeout args through to child repos. - "--quiet", - str(rctx.attr.quiet), - "--timeout", - str(rctx.attr.timeout), - "--annotations", - annotations_file, - ] + requirements_txt = _locked_requirements(rctx) + args = [ + python_interpreter, + "-m", + "python.pip_install.extract_wheels.parse_requirements_to_bzl", + "--requirements_lock", + rctx.path(requirements_txt), + "--requirements_lock_label", + str(requirements_txt), + # pass quiet and timeout args through to child repos. + "--quiet", + str(rctx.attr.quiet), + "--timeout", + str(rctx.attr.timeout), + "--annotations", + annotations_file, + ] - args += ["--python_interpreter", _get_python_interpreter_attr(rctx)] - if rctx.attr.python_interpreter_target: - args += ["--python_interpreter_target", str(rctx.attr.python_interpreter_target)] - progress_message = "Parsing requirements to starlark" - else: - args = [ - python_interpreter, - "-m", - "python.pip_install.extract_wheels.extract_wheels", - "--requirements", - rctx.path(rctx.attr.requirements), - "--annotations", - annotations_file, - ] - progress_message = "Extracting wheels" + args += ["--python_interpreter", _get_python_interpreter_attr(rctx)] + if rctx.attr.python_interpreter_target: + args += ["--python_interpreter_target", str(rctx.attr.python_interpreter_target)] + progress_message = "Parsing requirements to starlark" args += ["--repo", rctx.attr.name, "--repo-prefix", rctx.attr.repo_prefix] args = _parse_optional_attrs(rctx, args) @@ -253,13 +280,6 @@ common_env = [ ] common_attrs = { - "download_only": attr.bool( - doc = """ -Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of ---platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different -platform from the host platform. - """, - ), "enable_implicit_namespace_pkgs": attr.bool( default = False, doc = """ @@ -318,12 +338,7 @@ python_interpreter. ), "repo_prefix": attr.string( doc = """ -Prefix for the generated packages. For non-incremental mode the -packages will be of the form - -@///... - -For incremental mode the packages will be of the form +Prefix for the generated packages will be of the form @//... """, @@ -344,14 +359,6 @@ pip_repository_attrs = { "annotations": attr.string_dict( doc = "Optional annotations to apply to packages", ), - "incremental": attr.bool( - default = False, - doc = "Create the repository in incremental mode.", - ), - "requirements": attr.label( - allow_single_file = True, - doc = "A 'requirements.txt' pip requirements file.", - ), "requirements_darwin": attr.label( allow_single_file = True, doc = "Override the requirements_lock attribute when the host platform is Mac OS", diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl index bdd76b17d4..e42bb8e5ed 100644 --- a/python/pip_install/private/srcs.bzl +++ b/python/pip_install/private/srcs.bzl @@ -12,7 +12,6 @@ PIP_INSTALL_PY_SRCS = [ "@rules_python//python/pip_install/extract_wheels:arguments.py", "@rules_python//python/pip_install/extract_wheels:bazel.py", "@rules_python//python/pip_install/extract_wheels:extract_single_wheel.py", - "@rules_python//python/pip_install/extract_wheels:extract_wheels.py", "@rules_python//python/pip_install/extract_wheels:namespace_pkgs.py", "@rules_python//python/pip_install/extract_wheels:parse_requirements_to_bzl.py", "@rules_python//python/pip_install/extract_wheels:requirements.py", diff --git a/python/repositories.bzl b/python/repositories.bzl index f897904e70..dc2c49e722 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -37,6 +37,30 @@ def py_repositories(): STANDALONE_INTERPRETER_FILENAME = "STANDALONE_INTERPRETER" +def is_standalone_interpreter(rctx, python_interpreter_target): + """Query a python interpreter target for whether or not it's a rules_rust provided toolchain + + Args: + rctx (repository_ctx): The repository rule's context object. + python_interpreter_target (Target): A target representing a python interpreter. + + Returns: + bool: Whether or not the target is from a rules_python generated toolchain. + """ + + # Only update the location when using a hermetic toolchain. + if not python_interpreter_target: + return False + + # This is a rules_python provided toolchain. + return rctx.execute([ + "ls", + "{}/{}".format( + rctx.path(Label("@{}//:WORKSPACE".format(rctx.attr.python_interpreter_target.workspace_name))).dirname, + STANDALONE_INTERPRETER_FILENAME, + ), + ]).return_code == 0 + def _python_repository_impl(rctx): if rctx.attr.distutils and rctx.attr.distutils_content: fail("Only one of (distutils, distutils_content) should be set.") @@ -117,7 +141,7 @@ def _python_repository_impl(rctx): exec_result.stderr, ) fail(fail_msg) - exec_result = rctx.execute(["touch", "lib/.test"]) + exec_result = rctx.execute(["touch", "{}/.test".format(lib_dir)]) if exec_result.return_code == 0: exec_result = rctx.execute(["id", "-u"]) if exec_result.return_code != 0: diff --git a/tests/BUILD b/tests/BUILD index 46d8739622..b37a5a4232 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -8,8 +8,3 @@ bazel_integration_test( name = "pip_repository_entry_points_example", timeout = "long", ) - -bazel_integration_test( - name = "pip_deps_example", - timeout = "long", -) diff --git a/tests/pip_deps/BUILD b/tests/pip_deps/BUILD deleted file mode 100644 index 6a0412c718..0000000000 --- a/tests/pip_deps/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -filegroup( - name = "requirements_txt", - srcs = ["requirements.txt"], - visibility = ["//visibility:public"], -) - -test_suite( - name = "external_tests", - tests = [ - "@unpinned_pip//:pin_test", - ], -) - -sh_test( - name = "no_external_test", - srcs = ["test.sh"], - args = ["$(rootpath :requirements_txt)"], - data = [":requirements_txt"], -) diff --git a/tests/pip_deps/README.md b/tests/pip_deps/README.md deleted file mode 100644 index d9151baf11..0000000000 --- a/tests/pip_deps/README.md +++ /dev/null @@ -1 +0,0 @@ -Run `bazel run @unpinned_pip//:pin.update` to keep `PIP_PACKAGES` in `//:WORKSPACE` in sync with `//:requirements.txt`. diff --git a/tests/pip_deps/WORKSPACE b/tests/pip_deps/WORKSPACE deleted file mode 100644 index 83ecf85e37..0000000000 --- a/tests/pip_deps/WORKSPACE +++ /dev/null @@ -1,26 +0,0 @@ -workspace(name = "pip_deps") - -local_repository( - name = "rules_python", - path = "../..", -) - -load("@rules_python//python:repositories.bzl", "python_register_toolchains") - -# This toolchain is explicitly 3.10 while `rules_python` is 3.9 to act as -# a regression test, ensuring 3.10 is functional -python_register_toolchains( - name = "python310", - python_version = "3.10", -) - -load("@python310//:defs.bzl", "interpreter") -load("//:pip_deps.bzl", "pip_deps") - -PIP_PACKAGES = {"sphinx": "4.5.0"} - -pip_deps( - name = "pip", - packages = PIP_PACKAGES, - python_interpreter_target = interpreter, -) diff --git a/tests/pip_deps/pip_deps.bzl b/tests/pip_deps/pip_deps.bzl deleted file mode 100644 index 93426a6911..0000000000 --- a/tests/pip_deps/pip_deps.bzl +++ /dev/null @@ -1,55 +0,0 @@ -""" A demo implementation for pip_deps which provides @unpinned_pip//:pin. """ - -load("@rules_python//python:pip.bzl", "pip_parse") - -def _requirements_in_impl(repository_ctx): - repository_ctx.file( - "requirements.in", - content = "".join(["{package} == {version}\n".format( - package = package, - version = version, - ) for (package, version) in repository_ctx.attr.packages.items()]), - ) - repository_ctx.file("WORKSPACE", content = "") - repository_ctx.file("BUILD", content = """ -load("@rules_python//python:pip.bzl", "compile_pip_requirements") - -compile_pip_requirements( - name = "pin", - extra_args = ["--allow-unsafe"], - requirements_in = "requirements.in", - requirements_txt = "@{workspace_name}//{package}:{name}", -) -""".format( - workspace_name = repository_ctx.attr.requirements_lock.workspace_name, - package = repository_ctx.attr.requirements_lock.package, - name = repository_ctx.attr.requirements_lock.name, - )) - -_requirements_in = repository_rule( - implementation = _requirements_in_impl, - attrs = { - "packages": attr.string_dict(), - "requirements_lock": attr.label(allow_single_file = True), - }, -) - -def pip_deps( - *, - name = "pip", - packages = {}, - requirements_lock_target = Label("//:requirements_txt"), - requirements_lock_file = Label("//:requirements.txt"), - python_interpreter_target = None, - **kwargs): - _requirements_in( - name = "unpinned_" + name, - packages = packages, - requirements_lock = requirements_lock_target, - ) - pip_parse( - name = name, - requirements_lock = requirements_lock_file, - python_interpreter_target = python_interpreter_target, - **kwargs - ) diff --git a/tests/pip_deps/requirements.txt b/tests/pip_deps/requirements.txt deleted file mode 100644 index d5fa9e7254..0000000000 --- a/tests/pip_deps/requirements.txt +++ /dev/null @@ -1,136 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: -# -# bazel run //:pin.update -# -alabaster==0.7.12 \ - --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \ - --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 - # via sphinx -babel==2.10.1 \ - --hash=sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2 \ - --hash=sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13 - # via sphinx -certifi==2021.10.8 \ - --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ - --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 - # via requests -charset-normalizer==2.0.12 \ - --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 \ - --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df - # via requests -docutils==0.17.1 \ - --hash=sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125 \ - --hash=sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61 - # via sphinx -idna==3.3 \ - --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ - --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d - # via requests -imagesize==1.3.0 \ - --hash=sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c \ - --hash=sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d - # via sphinx -jinja2==3.1.2 \ - --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ - --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 - # via sphinx -markupsafe==2.1.1 \ - --hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \ - --hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \ - --hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \ - --hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \ - --hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \ - --hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \ - --hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \ - --hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \ - --hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \ - --hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \ - --hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \ - --hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \ - --hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \ - --hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \ - --hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \ - --hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \ - --hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \ - --hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \ - --hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \ - --hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \ - --hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \ - --hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \ - --hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \ - --hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \ - --hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \ - --hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \ - --hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \ - --hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \ - --hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \ - --hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \ - --hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \ - --hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \ - --hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \ - --hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \ - --hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \ - --hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \ - --hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \ - --hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \ - --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ - --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 - # via jinja2 -packaging==21.3 \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 - # via sphinx -pygments==2.12.0 \ - --hash=sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb \ - --hash=sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519 - # via sphinx -pyparsing==3.0.8 \ - --hash=sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954 \ - --hash=sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06 - # via packaging -pytz==2022.1 \ - --hash=sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7 \ - --hash=sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c - # via babel -requests==2.27.1 \ - --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \ - --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d - # via sphinx -snowballstemmer==2.2.0 \ - --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ - --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a - # via sphinx -sphinx==4.5.0 \ - --hash=sha256:7bf8ca9637a4ee15af412d1a1d9689fec70523a68ca9bb9127c2f3eeb344e2e6 \ - --hash=sha256:ebf612653238bcc8f4359627a9b7ce44ede6fdd75d9d30f68255c7383d3a6226 - # via -r requirements.in -sphinxcontrib-applehelp==1.0.2 \ - --hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \ - --hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58 - # via sphinx -sphinxcontrib-devhelp==1.0.2 \ - --hash=sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e \ - --hash=sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 \ - --hash=sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07 \ - --hash=sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2 - # via sphinx -sphinxcontrib-jsmath==1.0.1 \ - --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \ - --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8 - # via sphinx -sphinxcontrib-qthelp==1.0.3 \ - --hash=sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72 \ - --hash=sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 \ - --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \ - --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952 - # via sphinx -urllib3==1.26.9 \ - --hash=sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14 \ - --hash=sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e - # via requests diff --git a/tests/pip_deps/test.sh b/tests/pip_deps/test.sh deleted file mode 100755 index 5c356fd500..0000000000 --- a/tests/pip_deps/test.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -euo pipefail - -! grep external "$1" diff --git a/tests/pip_repository_entry_points/WORKSPACE b/tests/pip_repository_entry_points/WORKSPACE index 07a5d3aad0..dd80db47fa 100644 --- a/tests/pip_repository_entry_points/WORKSPACE +++ b/tests/pip_repository_entry_points/WORKSPACE @@ -24,9 +24,9 @@ pip_parse( requirements_lock = "//:requirements.txt", ) -load("@pip_parsed//:requirements.bzl", "install_deps") +load("@pip_parsed//:requirements.bzl", install_pip_parse_deps = "install_deps") -install_deps() +install_pip_parse_deps() # For a more thorough example of `pip_install`. See `@rules_python//examples/pip_install` pip_install( @@ -34,3 +34,7 @@ pip_install( python_interpreter_target = interpreter, requirements = "//:requirements.txt", ) + +load("@pip_installed//:requirements.bzl", install_pip_install_deps = "install_deps") + +install_pip_install_deps() diff --git a/tests/pip_repository_entry_points/requirements.txt b/tests/pip_repository_entry_points/requirements.txt index 02bc8751b0..279aed0e80 100644 --- a/tests/pip_repository_entry_points/requirements.txt +++ b/tests/pip_repository_entry_points/requirements.txt @@ -173,7 +173,7 @@ snowballstemmer==2.2.0 \ sphinx==4.3.2 \ --hash=sha256:0a8836751a68306b3fe97ecbe44db786f8479c3bf4b80e3a7f5c838657b4698c \ --hash=sha256:6a11ea5dd0bdb197f9c2abc2e0ce73e01340464feaece525e64036546d24c851 - # via -r requirements.in + # via -r ./requirements.in sphinxcontrib-applehelp==1.0.2 \ --hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \ --hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58 @@ -204,13 +204,13 @@ urllib3==1.26.7 \ # via requests yamllint==1.26.3 \ --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e - # via -r requirements.in + # via -r ./requirements.in # The following packages are considered to be unsafe in a requirements file: setuptools==59.6.0 \ --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e # via - # -r requirements.in + # -r ./requirements.in # sphinx # yamllint From b4f1d4270650d46d8f67c51c6e293aa5c3a672f2 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Tue, 30 Aug 2022 20:09:36 +1000 Subject: [PATCH 2/8] Update python/pip.bzl Co-authored-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/pip.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pip.bzl b/python/pip.bzl index 2419660b5f..d7287a8aa8 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -127,9 +127,9 @@ def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_ # Just in case our dependencies weren't already fetched pip_install_dependencies() - # Temporary compatibility shim - # pip_install was previously document to use requirements while pip_parse was using requirements_lock - # We would prefer everyone move to using requirements_lock, but we maintain a temporary shim + # Temporary compatibility shim. + # pip_install was previously document to use requirements while pip_parse was using requirements_lock. + # We would prefer everyone move to using requirements_lock, but we maintain a temporary shim. reqs_to_use = requirements_lock if requirements_lock else requirements pip_repository( From 6be2498bdc02568607f1502d0969bd5831fc06c5 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Wed, 31 Aug 2022 20:21:49 +1000 Subject: [PATCH 3/8] Updates after initial review. --- docs/pip.md | 22 +++++++++++++++++++--- python/pip.bzl | 28 ++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/pip.md b/docs/pip.md index b302ccfbeb..0930a7d00f 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -73,6 +73,22 @@ Annotations to apply to the BUILD file content from package generated from a `pi pip_install(requirements, name, kwargs) +Accepts a locked/compiled requirements file and installs the dependencies listed within. + +Usage: + +```python +load("@rules_python//python:pip.bzl", "pip_install") + +pip_install( + name = "pip_deps", + requirements = ":requirements.txt", +) + +load("@pip_deps//:requirements.bzl", "install_deps") + +install_deps() +``` **PARAMETERS** @@ -80,9 +96,9 @@ pip_install(requirements, -

| None | -| name |

-

| "pip" | -| kwargs |

-

| none | +| requirements | A 'requirements.txt' pip requirements file. | None | +| name | A unique name for the created external repository (default 'pip'). | "pip" | +| kwargs | Additional arguments to the [pip_repository](./pip_repository.md) repository rule. | none | diff --git a/python/pip.bzl b/python/pip.bzl index 2419660b5f..47dfe5f7a1 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -21,8 +21,32 @@ compile_pip_requirements = _compile_pip_requirements package_annotation = _package_annotation def pip_install(requirements = None, name = "pip", **kwargs): - # pip_install is now considered deprecated. - # In future, this may log a warning and eventually be removed. + """Accepts a locked/compiled requirements file and installs the dependencies listed within. + + Usage: + + ```python + load("@rules_python//python:pip.bzl", "pip_install") + + pip_install( + name = "pip_deps", + requirements = ":requirements.txt", + ) + + load("@pip_deps//:requirements.bzl", "install_deps") + + install_deps() + ``` + + Args: + requirements (Label): A 'requirements.txt' pip requirements file. + name (str, optional): A unique name for the created external repository (default 'pip'). + **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule. + + Deprecated: + Please use pip_parse. + """ + print("pip_install is deprecated. Please switch to pip_parse. pip_install will be removed in a future release.") pip_parse(requirements = requirements, name = name, **kwargs) def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_deps", **kwargs): From faf4ce81c5023cc483d024e61ae00a5393dee825 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Wed, 31 Aug 2022 20:34:50 +1000 Subject: [PATCH 4/8] Updates after initial review. --- python/pip.bzl | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/pip.bzl b/python/pip.bzl index c8669f6b2d..6a7a8f091d 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -42,9 +42,6 @@ def pip_install(requirements = None, name = "pip", **kwargs): requirements (Label): A 'requirements.txt' pip requirements file. name (str, optional): A unique name for the created external repository (default 'pip'). **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule. - - Deprecated: - Please use pip_parse. """ print("pip_install is deprecated. Please switch to pip_parse. pip_install will be removed in a future release.") pip_parse(requirements = requirements, name = name, **kwargs) From 617b4d0265c91f5f647f580125df691f614ca288 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Wed, 31 Aug 2022 20:38:32 +1000 Subject: [PATCH 5/8] Updates after initial review. --- docs/pip.md | 2 -- python/pip.bzl | 2 -- 2 files changed, 4 deletions(-) diff --git a/docs/pip.md b/docs/pip.md index 0930a7d00f..f6d8430adc 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -75,8 +75,6 @@ pip_install(requirements, Date: Wed, 31 Aug 2022 20:42:23 +1000 Subject: [PATCH 6/8] Updates after initial review. --- python/pip.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pip.bzl b/python/pip.bzl index b76e9093af..f34acaf351 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -41,6 +41,7 @@ def pip_install(requirements = None, name = "pip", **kwargs): name (str, optional): A unique name for the created external repository (default 'pip'). **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule. """ + # buildifier: disable=print print("pip_install is deprecated. Please switch to pip_parse. pip_install will be removed in a future release.") pip_parse(requirements = requirements, name = name, **kwargs) From cc3a096ea58478f62e7fbef100943f3072243424 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Wed, 31 Aug 2022 20:43:39 +1000 Subject: [PATCH 7/8] Updates after initial review. --- python/pip.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pip.bzl b/python/pip.bzl index f34acaf351..dfafefe38d 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -41,6 +41,7 @@ def pip_install(requirements = None, name = "pip", **kwargs): name (str, optional): A unique name for the created external repository (default 'pip'). **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule. """ + # buildifier: disable=print print("pip_install is deprecated. Please switch to pip_parse. pip_install will be removed in a future release.") pip_parse(requirements = requirements, name = name, **kwargs) From faae18778a5dd23a4e94f21fe2d912165799c014 Mon Sep 17 00:00:00 2001 From: Greg Date: Fri, 23 Sep 2022 09:29:36 +1000 Subject: [PATCH 8/8] Include notice about resolving deps for pip_install --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e0335241d3..944493c6a9 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,9 @@ updating your system `python` interpreter), you can force it to re-execute by ru Note: The `pip_install` rule is deprecated. `pip_parse` offers identical functionality and both `pip_install` and `pip_parse` now have the same implementation. The name `pip_install` may be removed in a future version of the rules. +The maintainers have taken all reasonable efforts to faciliate a smooth transition, but some users of `pip_install` will +need to replace their existing `requirements.txt` with a fully resolved set of dependencies using a tool such as +`pip-tools` or the `compile_pip_requirements` repository rule. ### Using third_party packages as dependencies