diff --git a/python-rpm-generators.spec b/python-rpm-generators.spec index 1e01d79..1992873 100644 --- a/python-rpm-generators.spec +++ b/python-rpm-generators.spec @@ -1,7 +1,7 @@ Name: python-rpm-generators Summary: Dependency generators for Python RPMs Version: 12 -Release: 1%{?dist} +Release: 2%{?dist} # Originally all those files were part of RPM, so license is kept here License: GPLv2+ @@ -21,7 +21,7 @@ BuildArch: noarch %package -n python3-rpm-generators Summary: %{summary} -Requires: python3-setuptools +Requires: python3-packaging # We have parametric macro generators, we need RPM 4.16 (4.15.90+ is 4.16 alpha) Requires: rpm > 4.15.90-0 # This contains the Lua functions we use: @@ -47,6 +47,11 @@ install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} *.py %{_rpmconfigdir}/pythonbundles.py %changelog +* Wed Feb 17 2021 Tomas Orsava - 12-2 +- scripts/pythondistdeps: Switch from using pkg_resources to importlib.metadata + for reading the egg/dist-info metadata +- The script no longer requires setuptools but instead requires packaging + * Wed Feb 03 2021 Miro Hrončok - 12-1 - Disable the dist generators for Python 2 - https://fedoraproject.org/wiki/Changes/Disable_Python_2_Dist_RPM_Generators_and_Freeze_Python_2_Macros diff --git a/pythondistdeps.py b/pythondistdeps.py index 70e08b8..dcc0a2a 100755 --- a/pythondistdeps.py +++ b/pythondistdeps.py @@ -11,21 +11,85 @@ # RPM python dependency generator, using .egg-info/.egg-link/.dist-info data # -# Please know: -# - Notes from an attempted rewrite from pkg_resources to importlib.metadata in -# 2020 can be found in the message of the commit that added this line. - from __future__ import print_function import argparse -from os.path import basename, dirname, isdir, sep -from sys import argv, stdin, stderr, version from distutils.sysconfig import get_python_lib +from os.path import dirname, sep +import re +from sys import argv, stdin, stderr from warnings import warn +from packaging.requirements import Requirement as Requirement_ +from packaging.version import parse + +try: + from importlib.metadata import PathDistribution +except ImportError: + from importlib_metadata import PathDistribution + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +def normalize_name(name): + """https://www.python.org/dev/peps/pep-0503/#normalized-names""" + return re.sub(r'[-_.]+', '-', name).lower() + + +def legacy_normalize_name(name): + """Like pkg_resources Distribution.key property""" + return re.sub(r'[-_]+', '-', name).lower() + + +class Requirement(Requirement_): + def __init__(self, requirement_string): + super(Requirement, self).__init__(requirement_string) + self.normalized_name = normalize_name(self.name) + self.legacy_normalized_name = legacy_normalize_name(self.name) + + +class Distribution(PathDistribution): + def __init__(self, path): + super(Distribution, self).__init__(Path(path)) + self.name = self.metadata['Name'] + self.normalized_name = normalize_name(self.name) + self.legacy_normalized_name = legacy_normalize_name(self.name) + self.requirements = [Requirement(r) for r in self.requires or []] + self.extras = [ + v for k, v in self.metadata.items() if k == 'Provides-Extra'] + self.py_version = self._parse_py_version(path) + + def _parse_py_version(self, path): + # Try to parse the Python version from the path the metadata + # resides at (e.g. /usr/lib/pythonX.Y/site-packages/...) + res = re.search(r"/python(?P\d+\.\d+)/", path) + if res: + return res.group('pyver') + # If that hasn't worked, attempt to parse it from the metadata + # directory name + res = re.search(r"-py(?P\d+.\d+)[.-]egg-info$", path) + if res: + return res.group('pyver') + return None + + def requirements_for_extra(self, extra): + extra_deps = [] + for req in self.requirements: + if not req.marker: + continue + if req.marker.evaluate(get_marker_env(self, extra)): + extra_deps.append(req) + return extra_deps + + def __repr__(self): + return '{} from {}'.format(self.name, self._path) + class RpmVersion(): def __init__(self, version_id): - version = parse_version(version_id) + version = parse(version_id) if isinstance(version._version, str): self.version = version._version else: @@ -144,10 +208,20 @@ def convert(name, operator, version_id): format(version_id, name)) from exc -def normalize_name(name): - """https://www.python.org/dev/peps/pep-0503/#normalized-names""" - import re - return re.sub(r'[-_.]+', '-', name).lower() +def get_marker_env(dist, extra): + # packaging uses a default environment using + # platform.python_version to evaluate if a dependency is relevant + # based on environment markers [1], + # e.g. requirement `argparse;python_version<"2.7"` + # + # Since we're running this script on one Python version while + # possibly evaluating packages for different versions, we + # set up an environment with the version we want to evaluate. + # + # [1] https://www.python.org/dev/peps/pep-0508/#environment-markers + return {"python_full_version": dist.py_version, + "python_version": dist.py_version, + "extra": extra} if __name__ == "__main__": @@ -243,52 +317,21 @@ if __name__ == "__main__": if lower.endswith('.egg') or \ lower.endswith('.egg-info') or \ lower.endswith('.dist-info'): - # This import is very slow, so only do it if needed - # - Notes from an attempted rewrite from pkg_resources to - # importlib.metadata in 2020 can be found in the message of - # the commit that added this line. - from pkg_resources import Distribution, FileMetadata, PathMetadata, Requirement, parse_version - dist_name = basename(f) - if isdir(f): - path_item = dirname(f) - metadata = PathMetadata(path_item, f) - else: - path_item = f - metadata = FileMetadata(f) - dist = Distribution.from_location(path_item, dist_name, metadata) - # Check if py_version is defined in the metadata file/directory name + dist = Distribution(f) if not dist.py_version: - # Try to parse the Python version from the path the metadata - # resides at (e.g. /usr/lib/pythonX.Y/site-packages/...) - import re - res = re.search(r"/python(?P\d+\.\d+)/", path_item) - if res: - dist.py_version = res.group('pyver') - else: - warn("Version for {!r} has not been found".format(dist), RuntimeWarning) - continue + warn("Version for {!r} has not been found".format(dist), RuntimeWarning) + continue - # pkg_resources use platform.python_version to evaluate if a - # dependency is relevant based on environment markers [1], - # e.g. requirement `argparse;python_version<"2.7"` - # - # Since we're running this script on one Python version while - # possibly evaluating packages for different versions, we mock the - # platform.python_version function. Discussed upstream [2]. - # - # [1] https://www.python.org/dev/peps/pep-0508/#environment-markers - # [2] https://github.com/pypa/setuptools/pull/1275 - import platform - platform.python_version = lambda: dist.py_version - platform.python_version_tuple = lambda: tuple(dist.py_version.split('.')) - - # This is the PEP 503 normalized name. - # It does also convert dots to dashes, unlike dist.key. - # See https://bugzilla.redhat.com/show_bug.cgi?id=1791530 - normalized_name = normalize_name(dist.project_name) - - # If we're processing an extras subpackage, check that the extras exists - if extras_subpackage and extras_subpackage not in dist.extras: + # If processing an extras subpackage: + # Check that the extras name is declared in the metadata, or + # that there are some dependencies associated with the extras + # name in the requires.txt (this is an outdated way to declare + # extras packages). + # - If there is an extras package declared only in requires.txt + # without any dependencies, this check will fail. In that case + # make sure to use updated metadata and declare the extras + # package there. + if extras_subpackage and extras_subpackage not in dist.extras and not dist.requirements_for_extra(extras_subpackage): print("*** PYTHON_EXTRAS_NOT_FOUND_ERROR___SEE_STDERR ***") print(f"\nError: The package name contains an extras name `{extras_subpackage}` that was not found in the metadata.\n" "Check if the extras were removed from the project. If so, consider removing the subpackage and obsoleting it from another.\n", file=stderr) @@ -301,32 +344,32 @@ if __name__ == "__main__": if args.provides: extras_suffix = f"[{extras_subpackage}]" if extras_subpackage else "" # If egg/dist metadata says package name is python, we provide python(abi) - if dist.key == 'python': + if dist.normalized_name == 'python': name = 'python(abi)' if name not in py_deps: py_deps[name] = [] py_deps[name].append(('==', dist.py_version)) if not args.legacy or not args.majorver_only: if normalized_names_provide_legacy: - name = 'python{}dist({}{})'.format(dist.py_version, dist.key, extras_suffix) + name = 'python{}dist({}{})'.format(dist.py_version, dist.legacy_normalized_name, extras_suffix) if name not in py_deps: py_deps[name] = [] if normalized_names_provide_pep503: - name_ = 'python{}dist({}{})'.format(dist.py_version, normalized_name, extras_suffix) + name_ = 'python{}dist({}{})'.format(dist.py_version, dist.normalized_name, extras_suffix) if name_ not in py_deps: py_deps[name_] = [] if args.majorver_provides or args.majorver_only or \ (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): if normalized_names_provide_legacy: - pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.key, extras_suffix) + pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.legacy_normalized_name, extras_suffix) if pymajor_name not in py_deps: py_deps[pymajor_name] = [] if normalized_names_provide_pep503: - pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, normalized_name, extras_suffix) + pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, dist.normalized_name, extras_suffix) if pymajor_name_ not in py_deps: py_deps[pymajor_name_] = [] if args.legacy or args.legacy_provides: - legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.key) + legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.legacy_normalized_name) if legacy_name not in py_deps: py_deps[legacy_name] = [] if dist.version: @@ -351,7 +394,7 @@ if __name__ == "__main__": if args.requires or (args.recommends and dist.extras): name = 'python(abi)' # If egg/dist metadata says package name is python, we don't add dependency on python(abi) - if dist.key == 'python': + if dist.normalized_name == 'python': py_abi = False if name in py_deps: py_deps.pop(name) @@ -361,24 +404,21 @@ if __name__ == "__main__": spec = ('==', dist.py_version) if spec not in py_deps[name]: py_deps[name].append(spec) - deps = dist.requires() - if args.recommends: - depsextras = dist.requires(extras=dist.extras) - if not args.requires: - for dep in reversed(depsextras): - if dep in deps: - depsextras.remove(dep) - deps = depsextras - elif extras_subpackage: - # Extras requires also contain the base requires included - deps = [d for d in dist.requires(extras=[extras_subpackage]) if d not in dist.requires()] + + if extras_subpackage: + deps = [d for d in dist.requirements_for_extra(extras_subpackage)] + else: + deps = dist.requirements + # console_scripts/gui_scripts entry points need pkg_resources from setuptools - if ((dist.get_entry_map('console_scripts') or - dist.get_entry_map('gui_scripts')) and + if (dist.entry_points and (lower.endswith('.egg') or lower.endswith('.egg-info'))): - # stick them first so any more specific requirement overrides it - deps.insert(0, Requirement.parse('setuptools')) + groups = {ep.group for ep in dist.entry_points} + if {"console_scripts", "gui_scripts"} & groups: + # stick them first so any more specific requirement + # overrides it + deps.insert(0, Requirement('setuptools')) # add requires/recommends based on egg/dist metadata for dep in deps: # Even if we're requiring `foo[bar]`, also require `foo` @@ -389,67 +429,62 @@ if __name__ == "__main__": # A dependency can have more than one extras, # i.e. foo[bar,baz], so let's go through all of them extras_suffixes += [f"[{e}]" for e in dep.extras] + for extras_suffix in extras_suffixes: if normalized_names_require_pep503: - dep_normalized_name = normalize_name(dep.project_name) + dep_normalized_name = dep.normalized_name else: - dep_normalized_name = dep.key + dep_normalized_name = dep.legacy_normalized_name if args.legacy: - name = 'pythonegg({})({})'.format(pyver_major, dep.key) + name = 'pythonegg({})({})'.format(pyver_major, dep.legacy_normalized_name) else: if args.majorver_only: name = 'python{}dist({}{})'.format(pyver_major, dep_normalized_name, extras_suffix) else: name = 'python{}dist({}{})'.format(dist.py_version, dep_normalized_name, extras_suffix) - for spec in dep.specs: - if name not in py_deps: - py_deps[name] = [] - if spec not in py_deps[name]: - py_deps[name].append(spec) - if not dep.specs: + + if dep.marker and not args.recommends and not extras_subpackage: + if not dep.marker.evaluate(get_marker_env(dist, '')): + continue + + if name not in py_deps: py_deps[name] = [] + for spec in dep.specifier: + if (spec.operator, spec.version) not in py_deps[name]: + py_deps[name].append((spec.operator, spec.version)) + # Unused, for automatic sub-package generation based on 'extras' from egg/dist metadata # TODO: implement in rpm later, or...? if args.extras: - deps = dist.requires() - extras = dist.extras - print(extras) - for extra in extras: + print(dist.extras) + for extra in dist.extras: print('%%package\textras-{}'.format(extra)) - print('Summary:\t{} extra for {} python package'.format(extra, dist.key)) + print('Summary:\t{} extra for {} python package'.format(extra, dist.legacy_normalized_name)) print('Group:\t\tDevelopment/Python') - depsextras = dist.requires(extras=[extra]) - for dep in reversed(depsextras): - if dep in deps: - depsextras.remove(dep) - deps = depsextras - for dep in deps: - for spec in dep.specs: - if spec[0] == '!=': - print('Conflicts:\t{} {} {}'.format(dep.key, '==', spec[1])) + for dep in dist.requirements_for_extra(extra): + for spec in dep.specifier: + if spec.operator == '!=': + print('Conflicts:\t{} {} {}'.format(dep.legacy_normalized_name, '==', spec.version)) else: - print('Requires:\t{} {} {}'.format(dep.key, spec[0], spec[1])) + print('Requires:\t{} {} {}'.format(dep.legacy_normalized_name, spec.operator, spec.version)) print('%%description\t{}'.format(extra)) - print('{} extra for {} python package'.format(extra, dist.key)) + print('{} extra for {} python package'.format(extra, dist.legacy_normalized_name)) print('%%files\t\textras-{}\n'.format(extra)) if args.conflicts: # Should we really add conflicts for extras? # Creating a meta package per extra with recommends on, which has # the requires/conflicts in stead might be a better solution... - for dep in dist.requires(extras=dist.extras): - name = dep.key - for spec in dep.specs: - if spec[0] == '!=': - if name not in py_deps: - py_deps[name] = [] - spec = ('==', spec[1]) - if spec not in py_deps[name]: - py_deps[name].append(spec) + for dep in dist.requirements: + for spec in dep.specifier: + if spec.operator == '!=': + if dep.legacy_normalized_name not in py_deps: + py_deps[dep.legacy_normalized_name] = [] + spec = ('==', spec.version) + if spec not in py_deps[dep.legacy_normalized_name]: + py_deps[dep.legacy_normalized_name].append(spec) - names = list(py_deps.keys()) - names.sort() - for name in names: + for name in sorted(py_deps): if py_deps[name]: # Print out versioned provides, requires, recommends, conflicts spec_list = [] diff --git a/tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt b/tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt index 22ef4d9..1436266 100644 --- a/tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt +++ b/tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt @@ -1,4 +1,3 @@ -# Taken from pyreq2rpm, removed tests that are expected to fail foobar0~=2.4.8 foobar1~=2.4.8.0 foobar2~=2.4.8.1 @@ -91,10 +90,8 @@ pyparsing0 pyparsing1>=2.0.1,!=2.0.4,!=2.1.2,!=2.1.6 babel>=1.3,!=2.0 -# Tests for breakages in Fedora fedora-python-nb2plots==0+unknown -# Other tests hugo1==1.0.0.dev7 hugo2<=8a4 hugo3!=11.1.1b14 diff --git a/tests/data/scripts_pythondistdeps/test-data.yaml b/tests/data/scripts_pythondistdeps/test-data.yaml index 6467c6c..9b97be8 100644 --- a/tests/data/scripts_pythondistdeps/test-data.yaml +++ b/tests/data/scripts_pythondistdeps/test-data.yaml @@ -1216,6 +1216,15 @@ python3dist(backports-range) = 3.7.2 python3dist(backports.range) = 3.7.2 requires: python(abi) = 3.7 +--requires --normalized-names-format pep503 --package-name python3-setuptools+certs: + --provides --majorver-provides --normalized-names-format pep503 --package-name python3-setuptools+certs: + usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info: + provides: |- + python3.9dist(setuptools[certs]) = 41.6 + python3dist(setuptools[certs]) = 41.6 + requires: |- + python(abi) = 3.9 + python3.9dist(certifi) = 2016.9.26 --requires --normalized-names-format pep503 --package-name python3-zope-component+testing: --provides --majorver-provides --normalized-names-format pep503 --package-name python3-zope-component+testing: usr/lib/python3.9/site-packages/zope.component-4.3.0-py3.9.egg-info: