From 77122ce501a7c6b08c08023914079167d946bf90 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 4 Dec 2024 15:58:57 +0100 Subject: [PATCH 1/8] =?UTF-8?q?=F0=9F=A7=AA=20Bump=20PyPy=20to=20v3.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is necessary because: * `pypy-3.8` is EOL * `pypy-3.8` has flaky SEGFAULTs on import [[1]] [[2]] [[3]] due to a bug in their GC that is fixed in `pypy-3.9` [1]: https://github.com/jazzband/pip-tools/actions/runs/12162197242/job/33918558133?pr=2106#step:8:59 [2]: https://github.com/pytest-dev/pytest/issues/11771#issuecomment-2200528806 [3]: https://pypy.org/posts/2024/03/fixing-bug-incremental-gc.html --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e166608..bf65d1cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,7 +148,7 @@ jobs: - MacOS - Windows python-version: - - pypy-3.8 + - pypy-3.9 pip-version: - latest env: From 822c3e6b55a414d1b404712deb8a4ded9665f09b Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 4 Dec 2024 16:36:02 +0100 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=A7=AA=20Stop=20running=20pip=20from?= =?UTF-8?q?=20`main`=20on=20PR/push?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ref #2140 --- .github/workflows/ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e166608..fb30342f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,15 +66,6 @@ jobs: || '["latest", "previous"]' ) }} - include: - - os: Ubuntu - python-version: >- - ${{ - github.job_workflow_sha - && '3.12-dev' - || '3.8' - }} - pip-version: main env: TOXENV: >- pip${{ matrix.pip-version }}${{ From fac71fcdf3cbf65ae696b807dae2ffeb19082097 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 4 Dec 2024 16:36:42 +0100 Subject: [PATCH 3/8] =?UTF-8?q?=F0=9F=A7=AA=F0=9F=90=9B=20Fix=20CI=20reusa?= =?UTF-8?q?ble=20workflow=20conditionals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs: * #2140 * https://github.com/actions/runner/issues/2417#issuecomment-1741105197 * https://github.com/actions/toolkit/issues/1264 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb30342f..c8c114ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: pip-version: >- ${{ fromJSON( - github.job_workflow_sha + inputs.cpython-pip-version && inputs.cpython-pip-version || '["latest", "previous"]' ) @@ -69,7 +69,7 @@ jobs: env: TOXENV: >- pip${{ matrix.pip-version }}${{ - !github.job_workflow_sha + !inputs.cpython-pip-version && '-coverage' || '' }} @@ -112,7 +112,7 @@ jobs: run: tox --skip-pkg-install - name: Upload coverage to Codecov if: >- - !github.job_workflow_sha + !inputs.cpython-pip-version uses: codecov/codecov-action@v3 with: files: ./coverage.xml From 6e8c21f77c69f938800ad0251b7e9c8c926c83d8 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 4 Dec 2024 16:47:06 +0100 Subject: [PATCH 4/8] =?UTF-8?q?=F0=9F=A7=AA=20Rename=20"previous"=20to=20"?= =?UTF-8?q?lowest"=20in=20tox?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This better reflects that the pip version being pulled in is the lowest tested/supported. --- .github/workflows/ci.yml | 2 +- .github/workflows/cron.yml | 2 +- tox.ini | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8c114ca..495d4069 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: fromJSON( inputs.cpython-pip-version && inputs.cpython-pip-version - || '["latest", "previous"]' + || '["latest", "lowest"]' ) }} env: diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index b024ea8c..ad348dae 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -11,4 +11,4 @@ jobs: uses: ./.github/workflows/ci.yml with: cpython-pip-version: >- - ["main", "latest", "previous"] + ["main", "latest", "lowest"] diff --git a/tox.ini b/tox.ini index bb740ee3..32a6e191 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = # NOTE: keep this in sync with the env list in .github/workflows/ci.yml. - py{38,39,310,311,312,py3}-pip{previous,latest,main}-coverage - pip{previous,latest,main}-coverage - pip{previous,latest,main} + py{38,39,310,311,312,py3}-pip{lowest,latest,main}-coverage + pip{lowest,latest,main}-coverage + pip{lowest,latest,main} checkqa readme skip_missing_interpreters = True @@ -14,7 +14,7 @@ extras = testing coverage: coverage deps = - pipprevious: pip==22.2.* + piplowest: pip==22.2.* piplatest: pip pipmain: https://github.com/pypa/pip/archive/main.zip setenv = From a49992b88c3e508ef259d0be3ce97e88234e5d38 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 4 Dec 2024 16:52:54 +0100 Subject: [PATCH 5/8] =?UTF-8?q?=F0=9F=A7=AA=20Add=20"supported"=20pip=20in?= =?UTF-8?q?=20tox=20for=20PR/push=20@=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is meant to be separate from "latest" and is pinned to the known working released version of pip. It will run on merges to `main` and in pull requests. And the cron runs will additionally test the latest version on PyPI and the `main` branch of the pip repo. --- .github/workflows/ci.yml | 4 ++-- .github/workflows/cron.yml | 2 +- tox.ini | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 495d4069..38baf3b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: fromJSON( inputs.cpython-pip-version && inputs.cpython-pip-version - || '["latest", "lowest"]' + || '["supported", "lowest"]' ) }} env: @@ -141,7 +141,7 @@ jobs: python-version: - pypy-3.8 pip-version: - - latest + - supported env: TOXENV: pip${{ matrix.pip-version }} steps: diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index ad348dae..23b0a218 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -11,4 +11,4 @@ jobs: uses: ./.github/workflows/ci.yml with: cpython-pip-version: >- - ["main", "latest", "lowest"] + ["main", "latest", "supported", "lowest"] diff --git a/tox.ini b/tox.ini index 32a6e191..85f3dc68 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = # NOTE: keep this in sync with the env list in .github/workflows/ci.yml. - py{38,39,310,311,312,py3}-pip{lowest,latest,main}-coverage - pip{lowest,latest,main}-coverage - pip{lowest,latest,main} + py{38,39,310,311,312,py3}-pip{supported,lowest,latest,main}-coverage + pip{supported,lowest,latest,main}-coverage + pip{supported,lowest,latest,main} checkqa readme skip_missing_interpreters = True @@ -14,6 +14,7 @@ extras = testing coverage: coverage deps = + pipsupported: pip==24.2 piplowest: pip==22.2.* piplatest: pip pipmain: https://github.com/pypa/pip/archive/main.zip From eeaef403de230d37d0f1ae52f0485419071ed0f9 Mon Sep 17 00:00:00 2001 From: chrysle <96722107+chrysle@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:59:16 +0200 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=90=9B=20Use=20requested=20constraint?= =?UTF-8?q?s=20for=20build=20backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes a bug that build deps compilation would get the latest version of an unconstrained build requirements list, not taking into account the restricted/requested one. The regression test is implemented against `setuptools < 70.1.0` which is known to inject a dependency on `wheel` (newer `setuptools` vendor it). The idea is that `pyproject.toml` does not have an upper bound for `setuptools` but the CLI arg does. And when this works correctly, the `wheel` entry will be included into the resolved output. Cleaning up `PIP_CONSTRAINT` is implemented manually due to the corner case of a permission error on Windows when accessing a file that we hold a file descriptor to from another subprocess[[1]]. It can be further simplified once the lowest Python version `pip-tools` supports is 3.12 by replacing `delete=False` with `delete_on_close=False` in the `tempfile.NamedTemporaryFile()` context manager initializer. [1]: https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile Co-authored-by: Sviatoslav Sydorenko --- piptools/build.py | 90 +++++++++++++++++++++++++++++++++++-- piptools/scripts/compile.py | 1 + tests/test_cli_compile.py | 72 +++++++++++++++++++++++++++-- 3 files changed, 157 insertions(+), 6 deletions(-) diff --git a/piptools/build.py b/piptools/build.py index c19e206b..3e7a5b2c 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -2,6 +2,7 @@ import collections import contextlib +import os import pathlib import sys import tempfile @@ -119,6 +120,7 @@ def build_project_metadata( src_file: pathlib.Path, build_targets: tuple[str, ...], *, + upgrade_packages: tuple[str, ...] | None = None, attempt_static_parse: bool, isolated: bool, quiet: bool, @@ -159,7 +161,12 @@ def build_project_metadata( return project_metadata src_dir = src_file.parent - with _create_project_builder(src_dir, isolated=isolated, quiet=quiet) as builder: + with _create_project_builder( + src_dir, + upgrade_packages=upgrade_packages, + isolated=isolated, + quiet=quiet, + ) as builder: metadata = _build_project_wheel_metadata(builder) extras = tuple(metadata.get_all("Provides-Extra") or ()) requirements = tuple( @@ -180,9 +187,80 @@ def build_project_metadata( ) +@contextlib.contextmanager +def _env_var( + env_var_name: str, + env_var_value: str, + /, +) -> Iterator[None]: + sentinel = object() + original_pip_constraint = os.getenv(env_var_name, sentinel) + pip_constraint_was_unset = original_pip_constraint is sentinel + + os.environ[env_var_name] = env_var_value + try: + yield + finally: + if pip_constraint_was_unset: + del os.environ[env_var_name] + return + + # Assert here is necessary because MyPy can't infer type + # narrowing in the complex case. + assert isinstance(original_pip_constraint, str) + os.environ[env_var_name] = original_pip_constraint + + +@contextlib.contextmanager +def _temporary_constraints_file_set_for_pip( + upgrade_packages: tuple[str, ...], +) -> Iterator[None]: + with tempfile.NamedTemporaryFile( + mode="w+t", + delete=False, # FIXME: switch to `delete_on_close` in Python 3.12+ + ) as tmpfile: + # NOTE: `delete_on_close=False` here (or rather `delete=False`, + # NOTE: temporarily) is important for cross-platform execution. It is + # NOTE: required on Windows so that the underlying `pip install` + # NOTE: invocation by pypa/build will be able to access the constraint + # NOTE: file via a subprocess and not fail installing it due to a + # NOTE: permission error related to this file handle still open in our + # NOTE: parent process. To achieve this, we `.close()` the file + # NOTE: descriptor before we hand off the control to the build frontend + # NOTE: and with `delete_on_close=False`, the + # NOTE: `tempfile.NamedTemporaryFile()` context manager does not remove + # NOTE: it from disk right away. + # NOTE: Due to support of versions below Python 3.12, we are forced to + # NOTE: temporarily resort to using `delete=False`, meaning that the CM + # NOTE: never attempts removing the file from disk, not even on exit. + # NOTE: So we do this manually until we can migrate to using the more + # NOTE: ergonomic argument `delete_on_close=False`. + + # Write packages to upgrade to a temporary file to set as + # constraints for the installation to the builder environment, + # in case build requirements are among them + tmpfile.write("\n".join(upgrade_packages)) + + # FIXME: replace `delete` with `delete_on_close` in Python 3.12+ + # FIXME: and replace `.close()` with `.flush()` + tmpfile.close() + + try: + with _env_var("PIP_CONSTRAINT", tmpfile.name): + yield + finally: + # FIXME: replace `delete` with `delete_on_close` in Python 3.12+ + # FIXME: and drop this manual deletion + os.unlink(tmpfile.name) + + @contextlib.contextmanager def _create_project_builder( - src_dir: pathlib.Path, *, isolated: bool, quiet: bool + src_dir: pathlib.Path, + *, + upgrade_packages: tuple[str, ...] | None = None, + isolated: bool, + quiet: bool, ) -> Iterator[build.ProjectBuilder]: if quiet: runner = pyproject_hooks.quiet_subprocess_runner @@ -193,7 +271,13 @@ def _create_project_builder( yield build.ProjectBuilder(src_dir, runner=runner) return - with build.env.DefaultIsolatedEnv() as env: + maybe_pip_constrained_context = ( + contextlib.nullcontext() + if upgrade_packages is None + else _temporary_constraints_file_set_for_pip(upgrade_packages) + ) + + with maybe_pip_constrained_context, build.env.DefaultIsolatedEnv() as env: builder = build.ProjectBuilder.from_isolated_env(env, src_dir, runner) env.install(builder.build_system_requires) env.install(builder.get_requires_for_build("wheel")) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 96aa9fa9..2b4522eb 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -370,6 +370,7 @@ def cli( metadata = build_project_metadata( src_file=Path(src_file), build_targets=build_deps_targets, + upgrade_packages=upgrade_packages, attempt_static_parse=not bool(build_deps_targets), isolated=build_isolation, quiet=log.verbosity <= 0, diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index c5031fc2..ad66dc3a 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -3431,7 +3431,6 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv """ ) ) - (tmp_path / "constraints.txt").write_text("wheel<0.43") out = runner.invoke( cli, [ @@ -3446,8 +3445,6 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv "--find-links", os.fspath(MINIMAL_WHEELS_PATH), os.fspath(tmp_path / "pyproject.toml"), - "--constraint", - os.fspath(tmp_path / "constraints.txt"), "--output-file", "-", ], @@ -3455,6 +3452,75 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv expected = rf"""foo[footest] @ {tmp_path.as_uri()} small-fake-a==0.2 small-fake-b==0.3 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +""" + try: + assert out.exit_code == 0 + assert expected == out.stdout + except Exception: # pragma: no cover + print(out.stdout) + print(out.stderr) + raise + + +@backtracking_resolver_only +def test_compile_build_targets_setuptools_no_wheel_dep( + runner, + tmp_path, + current_resolver, +): + """Check that user requests apply to build dependencies. + + This verifies that build deps compilation would not use the latest version + of an unconstrained build requirements list, when the user requested + restricting them. + + It is implemented against `setuptools < 70.1.0` which is known to inject a + dependency on `wheel` (newer `setuptools` vendor it). The idea is that + `pyproject.toml` does not have an upper bound for `setuptools` but the CLI + arg does. And when this works correctly, the `wheel` entry will be included + into the resolved output. + + This is a regression test for + https://github.com/jazzband/pip-tools/pull/1681#issuecomment-2212541289. + """ + (tmp_path / "pyproject.toml").write_text( + dedent( + """ + [project] + name = "foo" + version = "0.0.1" + dependencies = ["small-fake-a"] + """ + ) + ) + (tmp_path / "constraints.txt").write_text("wheel<0.43") + out = runner.invoke( + cli, + [ + "--build-isolation", + "--no-header", + "--no-annotate", + "--no-emit-options", + "--extra", + "dev", + "--build-deps-for", + "wheel", + "--find-links", + os.fspath(MINIMAL_WHEELS_PATH), + os.fspath(tmp_path / "pyproject.toml"), + "--constraint", + os.fspath(tmp_path / "constraints.txt"), + "--upgrade-package", + "setuptools < 70.1.0", # setuptools>=70.1.0 doesn't require wheel any more + "--output-file", + "-", + ], + catch_exceptions=True, + ) + expected = r"""small-fake-a==0.2 wheel==0.42.0 # The following packages are considered to be unsafe in a requirements file: From 94b0149f1921637fccf1d9895df1736295a14cab Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Mon, 4 Nov 2024 01:56:44 +0530 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=A7=AA=20Bump=20MyPy=20Python=20to=20?= =?UTF-8?q?3.9=20@=20pre-commit=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `pre-commit.ci` service dropped support for Python 3.8 as it's gone EOL. This change makes the check runnable again. It is suboptimal as we haven't yet dropped support for Python 3.8. #2144 will address the underlying issue more thoroughly. Resolves #2133. Co-Authored-By: Sviatoslav Sydorenko --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0aa16adf..f9a6a88b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: - build==1.0.0 - pyproject_hooks==1.0.0 - pytest==7.4.2 - language_version: python3.8 + language_version: python3.9 - repo: https://github.com/PyCQA/bandit rev: 1.7.8 hooks: From 01606eb7120153b67d079cf3f4a955d09907d2dd Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 16 Dec 2024 04:03:23 +0100 Subject: [PATCH 8/8] =?UTF-8?q?=F0=9F=A7=AA=20Bump=20PyPy=20to=20v3.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is necessary because: * `pypy-3.8` is EOL * `pypy-3.8` has flaky SEGFAULTs on import [[1]] [[2]] [[3]] due to a bug in their GC that is fixed in `pypy-3.9` * `pypy-3.9` has a bug that does not have a backport with the fix [[4]]. Furthermore, PyPy maintainers recommend using 3.10 [[5]]. [1]: https://github.com/jazzband/pip-tools/actions/runs/12162197242/job/33918558133?pr=2106#step:8:59 [2]: https://github.com/pytest-dev/pytest/issues/11771#issuecomment-2200528806 [3]: https://pypy.org/posts/2024/03/fixing-bug-incremental-gc.html [4]: https://github.com/tox-dev/tox/issues/3284 [5]: https://github.com/pypy/pypy/issues/4958#issuecomment-2326969261 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e166608..c82fb0c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,7 +148,7 @@ jobs: - MacOS - Windows python-version: - - pypy-3.8 + - pypy-3.10 pip-version: - latest env: