Don't generate runtime dependency on setuptools for console_scripts entrypoints

- recent setuptools don't need it, they use importlib.metadata
- for this to work, we need to detect setuptools version
- the detection is not bulletproof, but it fallbacks to the old behavior
This commit is contained in:
Miro Hrončok 2021-03-13 12:19:03 +01:00
parent a295a58559
commit 8485b55bea
7 changed files with 225 additions and 35 deletions

View File

@ -1,7 +1,7 @@
Name: python-rpm-generators
Summary: Dependency generators for Python RPMs
Version: 12
Release: 4%{?dist}
Release: 5%{?dist}
# Originally all those files were part of RPM, so license is kept here
License: GPLv2+
@ -47,6 +47,10 @@ install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} *.py
%{_rpmconfigdir}/pythonbundles.py
%changelog
* Sat Mar 13 2021 Miro Hrončok <mhroncok@redhat.com> - 12-5
- Don't generate runtime dependency on setuptools for console_scripts entrypoints,
recent setuptools don't need it
* Thu Mar 11 2021 Tomas Orsava <torsava@redhat.com> - 12-4
- scripts/pythondistdeps: Treat extras names case-insensitively and always
output them in lower case (#1936875)

View File

@ -16,7 +16,7 @@ import argparse
from distutils.sysconfig import get_python_lib
from os.path import dirname, sep
import re
from sys import argv, stdin, stderr
from sys import argv, stdin, stderr, version_info
from warnings import warn
from packaging.requirements import Requirement as Requirement_
@ -44,8 +44,12 @@ packaging.markers._operators["=="] = str_lower_eq
try:
from importlib.metadata import PathDistribution
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as version_of
except ImportError:
from importlib_metadata import PathDistribution
from importlib_metadata import PackageNotFoundError
from importlib_metadata import version as version_of
try:
from pathlib import Path
@ -252,6 +256,67 @@ def get_marker_env(dist, extra):
"extra": extra}
def find_setuptools_version(py_version):
"""Figure out setuptools version installed for the given Python X.Y version"""
# first, do it the sane way, if we run that Python version
# nb: this breaks if we run on PyPy and generate for CPython or vice versa
if '{}.{}'.format(*version_info[:2]) == py_version:
try:
return parse(version_of('setuptools'))
except PackageNotFoundError:
pass
# next, run the Python version and import setuptools from it
# nb: this also breaks if we run on different implementation
try:
import subprocess
code = 'import setuptools; print(setuptools.__version__)'
output = subprocess.check_output(['python{}'.format(dist.py_version), '-Esc', code],
universal_newlines=True)
return parse(output.strip())
except Exception: # anything can go wrong here
pass
warn('Cannot check setuptools version for Python {}; '
'assuming an old version for safety.'.format(py_version),
RuntimeWarning)
return parse('0')
def console_scripts_deps(dist, metadirname):
if metadirname.endswith('.dist-info'):
# console scripts from .dist-info have no extra deps
return []
groups = {ep.group for ep in dist.entry_points}
if not {"console_scripts", "gui_scripts"} & groups:
# a different kind of entrypoint
return []
python_version = parse(dist.py_version)
setuptools_version = find_setuptools_version(dist.py_version)
# https://setuptools.readthedocs.io/en/latest/history.html#v47-3-0
if setuptools_version >= parse('47.3.0'):
if python_version < parse('3.6'):
# don't use importlib_metadata on very old Pythons,
# use pkg_resources from setuptools
return [Requirement('setuptools')]
if python_version < parse('3.8'):
# technically this can also fallback to pkg_resources,
# but we prefer the lighter requirement
return [Requirement('importlib_metadata')]
return []
# https://setuptools.readthedocs.io/en/latest/history.html#v47-2-0
if setuptools_version >= parse('47.2.0') and python_version >= parse('3.8'):
return []
# older versions use pkg_resources from setuptools
return [Requirement('setuptools')]
if __name__ == "__main__":
"""To allow this script to be importable (and its classes/functions
reused), actions are performed only when run as a main script."""
@ -438,15 +503,12 @@ if __name__ == "__main__":
else:
deps = dist.requirements
# console_scripts/gui_scripts entry points need pkg_resources from setuptools
if (dist.entry_points and
(lower.endswith('.egg') or
lower.endswith('.egg-info'))):
groups = {ep.group for ep in dist.entry_points}
if {"console_scripts", "gui_scripts"} & groups:
# stick them first so any more specific requirement
# overrides it
deps.insert(0, Requirement('setuptools'))
# console_scripts/gui_scripts entry points might need extra dependencies
# we insert the requirement to the beginning of the list
# so any more specific requirement on the same package can override it
if dist.entry_points:
deps = console_scripts_deps(dist, lower) + deps
# add requires/recommends based on egg/dist metadata
for dep in deps:
# Even if we're requiring `foo[bar]`, also require `foo`

71
tests/console_script.sh Executable file
View File

@ -0,0 +1,71 @@
#!/usr/bin/bash -eux
RPMDIR=$(rpm --eval '%_topdir')/RPMS/noarch
RPMPKG="${RPMDIR}/isort-5.7.0-0.noarch.rpm"
mkdir -p $(rpm --eval '%_topdir')/SOURCES/
spectool -g -R isort.spec
for py_version in 3.6 3.7 3.8 3.9 3.10; do
rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec
# sanity check for provides; if this is broken, so is the test
rpm -qp --provides ${RPMPKG} | grep "python${py_version}dist(isort)"
# only the "main" Python version has setuptools installed,
# everything else does not
if [ "$py_version" = "$(rpm --eval '%python3_version')" ]; then
# all main Python/setuptools versions in Fedora 33+ are recent enough
# not to justify a generated dependency wrt the console_script
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true
else
# no setuptools installed, we assume an old version of setuptools was used to prepare the data
# hence the package always requires setuptools
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)"
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true
fi
# the rest is only possible on the CI or in mock/container, where we are root
# never run this script as root on your host OS, it is destructive
# also, it only works once (improvements welcome)
test $EUID -ne 0 && continue
export RPM_BUILD_ROOT=/
# install setuptools and build again
python${py_version} -m ensurepip
rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec
# the ensurepip version of setuptools is recent enough on Fedora 33+
# WARNING: Once we flip the rpmwheels bcond in python3.6, this assumption will break
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true
# but older Pythons still need importlib_metadata
if [[ "$py_version" = "3.6" || "$py_version" = "3.7" ]]; then
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)"
else
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true
fi
# install setuptools 47.2 and build again
python${py_version} -m pip uninstall --yes setuptools
python${py_version} -m pip install 'setuptools>=47.2.0,<47.3.0'
rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec
# this version of setuptools never uses importlib_metadata
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true
# older Pythons use setuptools, newer Pythons use importlib.metadata
if [[ "$py_version" = "3.6" || "$py_version" = "3.7" ]]; then
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)"
else
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)" && exit 1 || true
fi
# install an even older setuptools version and build again
python${py_version} -m pip uninstall --yes setuptools
python${py_version} -m pip install 'setuptools<47.2.0'
rpmbuild -ba --define "python3_test_version ${py_version}" isort.spec
# old console_scripts entyrpoint used pkg_resources only
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(setuptools)"
rpm -qp --requires ${RPMPKG} | grep "python${py_version}dist(importlib-metadata)" && exit 1 || true
done

View File

@ -131,7 +131,7 @@
python3dist(setuptools) = 41.6
requires: |-
python(abi) = 3.7
python3.7dist(setuptools)
python3.7dist(importlib-metadata)
usr/lib/python3.7/site-packages/setuptools-41.6.0.dist-info:
provides: |-
python3.7dist(setuptools) = 41.6
@ -233,9 +233,7 @@
provides: |-
python3.9dist(setuptools) = 41.6
python3dist(setuptools) = 41.6
requires: |-
python(abi) = 3.9
python3.9dist(setuptools)
requires: python(abi) = 3.9
usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info:
provides: |-
python3.9dist(setuptools) = 41.6
@ -360,9 +358,9 @@
python3dist(numpy-stl) = 2.11.2
requires: |-
python(abi) = 3.7
python3.7dist(importlib-metadata)
python3.7dist(numpy)
python3.7dist(python-utils) >= 1.6.2
python3.7dist(setuptools)
usr/lib64/python3.7/site-packages/scipy-1.2.1.dist-info:
provides: |-
python3.7dist(scipy) = 1.2.1
@ -395,7 +393,6 @@
python(abi) = 3.9
python3.9dist(numpy)
python3.9dist(python-utils) >= 1.6.2
python3.9dist(setuptools)
usr/lib64/python3.9/site-packages/simplejson-3.16.0-py3.9.egg-info:
provides: |-
python3.9dist(simplejson) = 3.16
@ -535,9 +532,7 @@
provides: |-
python3.9dist(setuptools) = 41.6
python3dist(setuptools) = 41.6
requires: |-
python(abi) = 3.9
python3.9dist(setuptools)
requires: python(abi) = 3.9
usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info:
provides: |-
python3.9dist(setuptools) = 41.6
@ -635,14 +630,12 @@
python2.7dist(zope.interface) >= 4.1
usr/lib/python3.10/site-packages/setuptools-41.6.0-py3.10.egg-info:
provides: python3.10dist(setuptools) = 41.6
requires: |-
python(abi) = 3.10
python3.10dist(setuptools)
requires: python(abi) = 3.10
usr/lib/python3.7/site-packages/setuptools-41.6.0-py3.7.egg-info:
provides: python3.7dist(setuptools) = 41.6
requires: |-
python(abi) = 3.7
python3.7dist(setuptools)
python3.7dist(importlib-metadata)
usr/lib/python3.7/site-packages/setuptools-41.6.0.dist-info:
provides: python3.7dist(setuptools) = 41.6
requires: python(abi) = 3.7
@ -697,9 +690,9 @@
provides: python3.7dist(numpy-stl) = 2.11.2
requires: |-
python(abi) = 3.7
python3.7dist(importlib-metadata)
python3.7dist(numpy)
python3.7dist(python-utils) >= 1.6.2
python3.7dist(setuptools)
usr/lib64/python3.7/site-packages/scipy-1.2.1.dist-info:
provides: python3.7dist(scipy) = 1.2.1
requires: |-
@ -728,7 +721,6 @@
python(abi) = 3.9
python3.9dist(numpy)
python3.9dist(python-utils) >= 1.6.2
python3.9dist(setuptools)
usr/lib64/python3.9/site-packages/simplejson-3.16.0-py3.9.egg-info:
provides: |-
python3.9dist(simplejson) = 3.16
@ -746,9 +738,7 @@
provides: |-
python3.10dist(setuptools) = 41.6
python3dist(setuptools) = 41.6
requires: |-
python(abi) = 3.10
python3.10dist(setuptools)
requires: python(abi) = 3.10
usr/lib/python3.11/site-packages/pip-20.0.2-py3.11.egg-info:
provides: python3.11dist(pip) = 20.0.2
requires: |-
@ -800,9 +790,7 @@
provides: |-
python3.10dist(setuptools) = 41.6
python3dist(setuptools) = 41.6
requires: |-
python(abi) = 3.10
python3.10dist(setuptools)
requires: python(abi) = 3.10
usr/lib/python3.11/site-packages/pip-20.0.2-py3.11.egg-info:
provides: |-
python3.11dist(pip) = 20.0.2

32
tests/isort.spec Normal file
View File

@ -0,0 +1,32 @@
Name: isort
Version: 5.7.0
Release: 0
Summary: A Python package with a console_scripts entrypoint
License: MIT
Source0: %{pypi_source}
BuildArch: noarch
BuildRequires: python3-devel
BuildRequires: python3-setuptools
BuildRequires: python%{python3_test_version}
%description
...
%prep
%autosetup
%build
%py3_build
%install
%py3_install
%if "%{python3_version}" != "%{python3_test_version}"
mv %{buildroot}%{_prefix}/lib/python%{python3_version} \
%{buildroot}%{_prefix}/lib/python%{python3_test_version}
mv %{buildroot}%{_prefix}/lib/python%{python3_test_version}/site-packages/%{name}-%{version}-py%{python3_version}.egg-info \
%{buildroot}%{_prefix}/lib/python%{python3_test_version}/site-packages/%{name}-%{version}-py%{python3_test_version}.egg-info
%endif
%files
%{_bindir}/%{name}*
%{_prefix}/lib/python%{python3_test_version}/site-packages/%{name}*

View File

@ -22,13 +22,15 @@
# Requirements for this script:
# - Python >= 3.6
# - pip >= 20.0.1
# - setuptools
# - setuptools >= 47.3
# - pytest
# - pyyaml
# - wheel
from pathlib import Path
import functools
import os
import pytest
import shlex
import shutil
@ -39,6 +41,7 @@ import yaml
PYTHONDISTDEPS_PATH = Path(__file__).parent / '..' / 'pythondistdeps.py'
TEST_DATA_PATH = Path(__file__).parent / 'data' / 'scripts_pythondistdeps'
FAKE_PATH = TEST_DATA_PATH / '_path'
def run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, expect_failure=False):
@ -47,10 +50,12 @@ def run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, exp
info_path = TEST_DATA_PATH / dist_egg_info_path
files = '\n'.join(map(str, info_path.iterdir()))
environ = fake_path_pythons()
provides = subprocess.run((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(provides_params)),
input=files, capture_output=True, check=False, encoding="utf-8")
input=files, capture_output=True, check=False, encoding="utf-8", env=environ)
requires = subprocess.run((sys.executable, PYTHONDISTDEPS_PATH, *shlex.split(requires_params)),
input=files, capture_output=True, check=False, encoding="utf-8")
input=files, capture_output=True, check=False, encoding="utf-8", env=environ)
if expect_failure:
if provides.returncode == 0 or requires.returncode == 0:
@ -67,6 +72,25 @@ def run_pythondistdeps(provides_params, requires_params, dist_egg_info_path, exp
return {"provides": provides.stdout.strip(), "requires": requires.stdout.strip()}
@functools.lru_cache(maxsize=1)
def fake_path_pythons():
"""Create fake Pythons, so when the pythondistdeps script tries to detect
setuptools version, it tells them ours."""
environ = os.environ.copy()
path = environ.get("PATH", "")
path = f"{FAKE_PATH}:{path}"
environ["PATH"] = path
FAKE_PATH.mkdir(exist_ok=True)
for ver in "2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10":
exe = FAKE_PATH / f"python{ver}"
exe.unlink(missing_ok=True)
exe.symlink_to(sys.executable)
return environ
def load_test_data():
"""Reads the test-data.yaml and loads the test data into a dict."""
with TEST_DATA_PATH.joinpath('test-data.yaml').open() as file:

View File

@ -34,6 +34,10 @@
dir: ./tests
# Use update-test-sources.sh to update the test data
run: python3 -m pytest --capture=no -vvv
# WARNING: This test alters the environment, keep it last:
- console_script:
dir: .
run: ./console_script.sh
required_packages:
- rpm-build
- rpmdevtools
@ -43,3 +47,8 @@
- python3-pyyaml
- python3-setuptools
- python3-wheel
- python3.6
- python3.7
- python3.8
- python3.9
- python3.10