From 438d8d3b705e88b62d4d36347338fd5357a71687 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Wed, 17 Feb 2021 12:18:26 +0100 Subject: [PATCH] scripts/pythondistdeps: Backport switch to importlib.metadata from upstream Upstream change to importlib.metadata: https://github.com/rpm-software-management/rpm/pull/1317 Due to extras packages being hadled slightly differently by importlib, one test case for this was added. And due to changes in handling requires.txt files, comments were removed from the pyreq2rpm.tests testing package. Also because of the switch, we removed the dependency on setuptools and added a dependency on packaging. Note: Some packages with egg-info files might provide a different name due to this change if there is a conflict between the filename and the name in the metadata. Previously, the filename was sometimes used to parse the name, now it is always the content of that file, which is what packaging does, and thus also pip and other Python tooling. Currently, this is known to affect only 1 package in Fedora (ntpsec). The resulting script is different from upstream because of not yet upstreamed changes in Fedora: - scripts/pythondistdeps: Rework error messages - scripts/pythondistdeps: Add parameter --package-name - scripts/pythondistdeps: Implement provides/requires for extras packages - pythondistdeps.py: When parsing extras name, take the rightmost + These changes are proposed in this upstream PR: https://github.com/rpm-software-management/rpm/pull/1546 --- python-rpm-generators.spec | 9 +- pythondistdeps.py | 263 ++++++++++-------- .../requires.txt | 3 - .../scripts_pythondistdeps/test-data.yaml | 9 + 4 files changed, 165 insertions(+), 119 deletions(-) diff --git a/python-rpm-generators.spec b/python-rpm-generators.spec index 1e01d79..1992873 100644 --- a/python-rpm-generators.spec +++ b/python-rpm-generators.spec @@ -1,7 +1,7 @@ Name: python-rpm-generators Summary: Dependency generators for Python RPMs Version: 12 -Release: 1%{?dist} +Release: 2%{?dist} # Originally all those files were part of RPM, so license is kept here License: GPLv2+ @@ -21,7 +21,7 @@ BuildArch: noarch %package -n python3-rpm-generators Summary: %{summary} -Requires: python3-setuptools +Requires: python3-packaging # We have parametric macro generators, we need RPM 4.16 (4.15.90+ is 4.16 alpha) Requires: rpm > 4.15.90-0 # This contains the Lua functions we use: @@ -47,6 +47,11 @@ install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} *.py %{_rpmconfigdir}/pythonbundles.py %changelog +* Wed Feb 17 2021 Tomas Orsava - 12-2 +- scripts/pythondistdeps: Switch from using pkg_resources to importlib.metadata + for reading the egg/dist-info metadata +- The script no longer requires setuptools but instead requires packaging + * Wed Feb 03 2021 Miro Hrončok - 12-1 - Disable the dist generators for Python 2 - https://fedoraproject.org/wiki/Changes/Disable_Python_2_Dist_RPM_Generators_and_Freeze_Python_2_Macros diff --git a/pythondistdeps.py b/pythondistdeps.py index 70e08b8..dcc0a2a 100755 --- a/pythondistdeps.py +++ b/pythondistdeps.py @@ -11,21 +11,85 @@ # RPM python dependency generator, using .egg-info/.egg-link/.dist-info data # -# Please know: -# - Notes from an attempted rewrite from pkg_resources to importlib.metadata in -# 2020 can be found in the message of the commit that added this line. - from __future__ import print_function import argparse -from os.path import basename, dirname, isdir, sep -from sys import argv, stdin, stderr, version from distutils.sysconfig import get_python_lib +from os.path import dirname, sep +import re +from sys import argv, stdin, stderr from warnings import warn +from packaging.requirements import Requirement as Requirement_ +from packaging.version import parse + +try: + from importlib.metadata import PathDistribution +except ImportError: + from importlib_metadata import PathDistribution + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +def normalize_name(name): + """https://www.python.org/dev/peps/pep-0503/#normalized-names""" + return re.sub(r'[-_.]+', '-', name).lower() + + +def legacy_normalize_name(name): + """Like pkg_resources Distribution.key property""" + return re.sub(r'[-_]+', '-', name).lower() + + +class Requirement(Requirement_): + def __init__(self, requirement_string): + super(Requirement, self).__init__(requirement_string) + self.normalized_name = normalize_name(self.name) + self.legacy_normalized_name = legacy_normalize_name(self.name) + + +class Distribution(PathDistribution): + def __init__(self, path): + super(Distribution, self).__init__(Path(path)) + self.name = self.metadata['Name'] + self.normalized_name = normalize_name(self.name) + self.legacy_normalized_name = legacy_normalize_name(self.name) + self.requirements = [Requirement(r) for r in self.requires or []] + self.extras = [ + v for k, v in self.metadata.items() if k == 'Provides-Extra'] + self.py_version = self._parse_py_version(path) + + def _parse_py_version(self, path): + # Try to parse the Python version from the path the metadata + # resides at (e.g. /usr/lib/pythonX.Y/site-packages/...) + res = re.search(r"/python(?P\d+\.\d+)/", path) + if res: + return res.group('pyver') + # If that hasn't worked, attempt to parse it from the metadata + # directory name + res = re.search(r"-py(?P\d+.\d+)[.-]egg-info$", path) + if res: + return res.group('pyver') + return None + + def requirements_for_extra(self, extra): + extra_deps = [] + for req in self.requirements: + if not req.marker: + continue + if req.marker.evaluate(get_marker_env(self, extra)): + extra_deps.append(req) + return extra_deps + + def __repr__(self): + return '{} from {}'.format(self.name, self._path) + class RpmVersion(): def __init__(self, version_id): - version = parse_version(version_id) + version = parse(version_id) if isinstance(version._version, str): self.version = version._version else: @@ -144,10 +208,20 @@ def convert(name, operator, version_id): format(version_id, name)) from exc -def normalize_name(name): - """https://www.python.org/dev/peps/pep-0503/#normalized-names""" - import re - return re.sub(r'[-_.]+', '-', name).lower() +def get_marker_env(dist, extra): + # packaging uses a default environment using + # platform.python_version to evaluate if a dependency is relevant + # based on environment markers [1], + # e.g. requirement `argparse;python_version<"2.7"` + # + # Since we're running this script on one Python version while + # possibly evaluating packages for different versions, we + # set up an environment with the version we want to evaluate. + # + # [1] https://www.python.org/dev/peps/pep-0508/#environment-markers + return {"python_full_version": dist.py_version, + "python_version": dist.py_version, + "extra": extra} if __name__ == "__main__": @@ -243,52 +317,21 @@ if __name__ == "__main__": if lower.endswith('.egg') or \ lower.endswith('.egg-info') or \ lower.endswith('.dist-info'): - # This import is very slow, so only do it if needed - # - Notes from an attempted rewrite from pkg_resources to - # importlib.metadata in 2020 can be found in the message of - # the commit that added this line. - from pkg_resources import Distribution, FileMetadata, PathMetadata, Requirement, parse_version - dist_name = basename(f) - if isdir(f): - path_item = dirname(f) - metadata = PathMetadata(path_item, f) - else: - path_item = f - metadata = FileMetadata(f) - dist = Distribution.from_location(path_item, dist_name, metadata) - # Check if py_version is defined in the metadata file/directory name + dist = Distribution(f) if not dist.py_version: - # Try to parse the Python version from the path the metadata - # resides at (e.g. /usr/lib/pythonX.Y/site-packages/...) - import re - res = re.search(r"/python(?P\d+\.\d+)/", path_item) - if res: - dist.py_version = res.group('pyver') - else: - warn("Version for {!r} has not been found".format(dist), RuntimeWarning) - continue + warn("Version for {!r} has not been found".format(dist), RuntimeWarning) + continue - # pkg_resources use platform.python_version to evaluate if a - # dependency is relevant based on environment markers [1], - # e.g. requirement `argparse;python_version<"2.7"` - # - # Since we're running this script on one Python version while - # possibly evaluating packages for different versions, we mock the - # platform.python_version function. Discussed upstream [2]. - # - # [1] https://www.python.org/dev/peps/pep-0508/#environment-markers - # [2] https://github.com/pypa/setuptools/pull/1275 - import platform - platform.python_version = lambda: dist.py_version - platform.python_version_tuple = lambda: tuple(dist.py_version.split('.')) - - # This is the PEP 503 normalized name. - # It does also convert dots to dashes, unlike dist.key. - # See https://bugzilla.redhat.com/show_bug.cgi?id=1791530 - normalized_name = normalize_name(dist.project_name) - - # If we're processing an extras subpackage, check that the extras exists - if extras_subpackage and extras_subpackage not in dist.extras: + # If processing an extras subpackage: + # Check that the extras name is declared in the metadata, or + # that there are some dependencies associated with the extras + # name in the requires.txt (this is an outdated way to declare + # extras packages). + # - If there is an extras package declared only in requires.txt + # without any dependencies, this check will fail. In that case + # make sure to use updated metadata and declare the extras + # package there. + if extras_subpackage and extras_subpackage not in dist.extras and not dist.requirements_for_extra(extras_subpackage): print("*** PYTHON_EXTRAS_NOT_FOUND_ERROR___SEE_STDERR ***") print(f"\nError: The package name contains an extras name `{extras_subpackage}` that was not found in the metadata.\n" "Check if the extras were removed from the project. If so, consider removing the subpackage and obsoleting it from another.\n", file=stderr) @@ -301,32 +344,32 @@ if __name__ == "__main__": if args.provides: extras_suffix = f"[{extras_subpackage}]" if extras_subpackage else "" # If egg/dist metadata says package name is python, we provide python(abi) - if dist.key == 'python': + if dist.normalized_name == 'python': name = 'python(abi)' if name not in py_deps: py_deps[name] = [] py_deps[name].append(('==', dist.py_version)) if not args.legacy or not args.majorver_only: if normalized_names_provide_legacy: - name = 'python{}dist({}{})'.format(dist.py_version, dist.key, extras_suffix) + name = 'python{}dist({}{})'.format(dist.py_version, dist.legacy_normalized_name, extras_suffix) if name not in py_deps: py_deps[name] = [] if normalized_names_provide_pep503: - name_ = 'python{}dist({}{})'.format(dist.py_version, normalized_name, extras_suffix) + name_ = 'python{}dist({}{})'.format(dist.py_version, dist.normalized_name, extras_suffix) if name_ not in py_deps: py_deps[name_] = [] if args.majorver_provides or args.majorver_only or \ (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions): if normalized_names_provide_legacy: - pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.key, extras_suffix) + pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.legacy_normalized_name, extras_suffix) if pymajor_name not in py_deps: py_deps[pymajor_name] = [] if normalized_names_provide_pep503: - pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, normalized_name, extras_suffix) + pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, dist.normalized_name, extras_suffix) if pymajor_name_ not in py_deps: py_deps[pymajor_name_] = [] if args.legacy or args.legacy_provides: - legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.key) + legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.legacy_normalized_name) if legacy_name not in py_deps: py_deps[legacy_name] = [] if dist.version: @@ -351,7 +394,7 @@ if __name__ == "__main__": if args.requires or (args.recommends and dist.extras): name = 'python(abi)' # If egg/dist metadata says package name is python, we don't add dependency on python(abi) - if dist.key == 'python': + if dist.normalized_name == 'python': py_abi = False if name in py_deps: py_deps.pop(name) @@ -361,24 +404,21 @@ if __name__ == "__main__": spec = ('==', dist.py_version) if spec not in py_deps[name]: py_deps[name].append(spec) - deps = dist.requires() - if args.recommends: - depsextras = dist.requires(extras=dist.extras) - if not args.requires: - for dep in reversed(depsextras): - if dep in deps: - depsextras.remove(dep) - deps = depsextras - elif extras_subpackage: - # Extras requires also contain the base requires included - deps = [d for d in dist.requires(extras=[extras_subpackage]) if d not in dist.requires()] + + if extras_subpackage: + deps = [d for d in dist.requirements_for_extra(extras_subpackage)] + else: + deps = dist.requirements + # console_scripts/gui_scripts entry points need pkg_resources from setuptools - if ((dist.get_entry_map('console_scripts') or - dist.get_entry_map('gui_scripts')) and + if (dist.entry_points and (lower.endswith('.egg') or lower.endswith('.egg-info'))): - # stick them first so any more specific requirement overrides it - deps.insert(0, Requirement.parse('setuptools')) + groups = {ep.group for ep in dist.entry_points} + if {"console_scripts", "gui_scripts"} & groups: + # stick them first so any more specific requirement + # overrides it + deps.insert(0, Requirement('setuptools')) # add requires/recommends based on egg/dist metadata for dep in deps: # Even if we're requiring `foo[bar]`, also require `foo` @@ -389,67 +429,62 @@ if __name__ == "__main__": # A dependency can have more than one extras, # i.e. foo[bar,baz], so let's go through all of them extras_suffixes += [f"[{e}]" for e in dep.extras] + for extras_suffix in extras_suffixes: if normalized_names_require_pep503: - dep_normalized_name = normalize_name(dep.project_name) + dep_normalized_name = dep.normalized_name else: - dep_normalized_name = dep.key + dep_normalized_name = dep.legacy_normalized_name if args.legacy: - name = 'pythonegg({})({})'.format(pyver_major, dep.key) + name = 'pythonegg({})({})'.format(pyver_major, dep.legacy_normalized_name) else: if args.majorver_only: name = 'python{}dist({}{})'.format(pyver_major, dep_normalized_name, extras_suffix) else: name = 'python{}dist({}{})'.format(dist.py_version, dep_normalized_name, extras_suffix) - for spec in dep.specs: - if name not in py_deps: - py_deps[name] = [] - if spec not in py_deps[name]: - py_deps[name].append(spec) - if not dep.specs: + + if dep.marker and not args.recommends and not extras_subpackage: + if not dep.marker.evaluate(get_marker_env(dist, '')): + continue + + if name not in py_deps: py_deps[name] = [] + for spec in dep.specifier: + if (spec.operator, spec.version) not in py_deps[name]: + py_deps[name].append((spec.operator, spec.version)) + # Unused, for automatic sub-package generation based on 'extras' from egg/dist metadata # TODO: implement in rpm later, or...? if args.extras: - deps = dist.requires() - extras = dist.extras - print(extras) - for extra in extras: + print(dist.extras) + for extra in dist.extras: print('%%package\textras-{}'.format(extra)) - print('Summary:\t{} extra for {} python package'.format(extra, dist.key)) + print('Summary:\t{} extra for {} python package'.format(extra, dist.legacy_normalized_name)) print('Group:\t\tDevelopment/Python') - depsextras = dist.requires(extras=[extra]) - for dep in reversed(depsextras): - if dep in deps: - depsextras.remove(dep) - deps = depsextras - for dep in deps: - for spec in dep.specs: - if spec[0] == '!=': - print('Conflicts:\t{} {} {}'.format(dep.key, '==', spec[1])) + for dep in dist.requirements_for_extra(extra): + for spec in dep.specifier: + if spec.operator == '!=': + print('Conflicts:\t{} {} {}'.format(dep.legacy_normalized_name, '==', spec.version)) else: - print('Requires:\t{} {} {}'.format(dep.key, spec[0], spec[1])) + print('Requires:\t{} {} {}'.format(dep.legacy_normalized_name, spec.operator, spec.version)) print('%%description\t{}'.format(extra)) - print('{} extra for {} python package'.format(extra, dist.key)) + print('{} extra for {} python package'.format(extra, dist.legacy_normalized_name)) print('%%files\t\textras-{}\n'.format(extra)) if args.conflicts: # Should we really add conflicts for extras? # Creating a meta package per extra with recommends on, which has # the requires/conflicts in stead might be a better solution... - for dep in dist.requires(extras=dist.extras): - name = dep.key - for spec in dep.specs: - if spec[0] == '!=': - if name not in py_deps: - py_deps[name] = [] - spec = ('==', spec[1]) - if spec not in py_deps[name]: - py_deps[name].append(spec) + for dep in dist.requirements: + for spec in dep.specifier: + if spec.operator == '!=': + if dep.legacy_normalized_name not in py_deps: + py_deps[dep.legacy_normalized_name] = [] + spec = ('==', spec.version) + if spec not in py_deps[dep.legacy_normalized_name]: + py_deps[dep.legacy_normalized_name].append(spec) - names = list(py_deps.keys()) - names.sort() - for name in names: + for name in sorted(py_deps): if py_deps[name]: # Print out versioned provides, requires, recommends, conflicts spec_list = [] diff --git a/tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt b/tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt index 22ef4d9..1436266 100644 --- a/tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt +++ b/tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt @@ -1,4 +1,3 @@ -# Taken from pyreq2rpm, removed tests that are expected to fail foobar0~=2.4.8 foobar1~=2.4.8.0 foobar2~=2.4.8.1 @@ -91,10 +90,8 @@ pyparsing0 pyparsing1>=2.0.1,!=2.0.4,!=2.1.2,!=2.1.6 babel>=1.3,!=2.0 -# Tests for breakages in Fedora fedora-python-nb2plots==0+unknown -# Other tests hugo1==1.0.0.dev7 hugo2<=8a4 hugo3!=11.1.1b14 diff --git a/tests/data/scripts_pythondistdeps/test-data.yaml b/tests/data/scripts_pythondistdeps/test-data.yaml index 6467c6c..9b97be8 100644 --- a/tests/data/scripts_pythondistdeps/test-data.yaml +++ b/tests/data/scripts_pythondistdeps/test-data.yaml @@ -1216,6 +1216,15 @@ python3dist(backports-range) = 3.7.2 python3dist(backports.range) = 3.7.2 requires: python(abi) = 3.7 +--requires --normalized-names-format pep503 --package-name python3-setuptools+certs: + --provides --majorver-provides --normalized-names-format pep503 --package-name python3-setuptools+certs: + usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info: + provides: |- + python3.9dist(setuptools[certs]) = 41.6 + python3dist(setuptools[certs]) = 41.6 + requires: |- + python(abi) = 3.9 + python3.9dist(certifi) = 2016.9.26 --requires --normalized-names-format pep503 --package-name python3-zope-component+testing: --provides --majorver-provides --normalized-names-format pep503 --package-name python3-zope-component+testing: usr/lib/python3.9/site-packages/zope.component-4.3.0-py3.9.egg-info: