scripts/pythondistdeps: Backport switch to importlib.metadata from upstream

Upstream change to importlib.metadata: https://github.com/rpm-software-management/rpm/pull/1317

Due to extras packages being hadled slightly differently by importlib,
one test case for this was added.  And due to changes in handling
requires.txt files, comments were removed from the pyreq2rpm.tests
testing package.

Also because of the switch, we removed the dependency on setuptools and
added a dependency on packaging.

Note: Some packages with egg-info files might provide a different name
due to this change if there is a conflict between the filename and the
name in the metadata. Previously, the filename was sometimes used to
parse the name, now it is always the content of that file, which is what
packaging does, and thus also pip and other Python tooling. Currently,
this is known to affect only 1 package in Fedora (ntpsec).

The resulting script is different from upstream because of not yet upstreamed changes in Fedora:
- scripts/pythondistdeps: Rework error messages
- scripts/pythondistdeps: Add parameter --package-name
- scripts/pythondistdeps: Implement provides/requires for extras packages
- pythondistdeps.py: When parsing extras name, take the rightmost +

These changes are proposed in this upstream PR: https://github.com/rpm-software-management/rpm/pull/1546

python-rpm-generators.spec

@@ -1,7 +1,7 @@
 Name:           python-rpm-generators
 Summary:        Dependency generators for Python RPMs
 Version:        12
-Release:        1%{?dist}
+Release:        2%{?dist}
 
 # Originally all those files were part of RPM, so license is kept here
 License:        GPLv2+
@@ -21,7 +21,7 @@ BuildArch:      noarch
 
 %package -n python3-rpm-generators
 Summary:        %{summary}
-Requires:       python3-setuptools
+Requires:       python3-packaging
 # We have parametric macro generators, we need RPM 4.16 (4.15.90+ is 4.16 alpha)
 Requires:       rpm > 4.15.90-0
 # This contains the Lua functions we use:
@@ -47,6 +47,11 @@ install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} *.py
 %{_rpmconfigdir}/pythonbundles.py
 
 %changelog
+* Wed Feb 17 2021 Tomas Orsava <torsava@redhat.com> - 12-2
+- scripts/pythondistdeps: Switch from using pkg_resources to importlib.metadata
+  for reading the egg/dist-info metadata
+- The script no longer requires setuptools but instead requires packaging
+
 * Wed Feb 03 2021 Miro Hrončok <mhroncok@redhat.com> - 12-1
 - Disable the dist generators for Python 2
 - https://fedoraproject.org/wiki/Changes/Disable_Python_2_Dist_RPM_Generators_and_Freeze_Python_2_Macros

pythondistdeps.py

@@ -11,21 +11,85 @@
 # RPM python dependency generator, using .egg-info/.egg-link/.dist-info data
 #
 
-# Please know:
-# - Notes from an attempted rewrite from pkg_resources to importlib.metadata in
-#   2020 can be found in the message of the commit that added this line.
-
 from __future__ import print_function
 import argparse
-from os.path import basename, dirname, isdir, sep
-from sys import argv, stdin, stderr, version
 from distutils.sysconfig import get_python_lib
+from os.path import dirname, sep
+import re
+from sys import argv, stdin, stderr
 from warnings import warn
 
+from packaging.requirements import Requirement as Requirement_
+from packaging.version import parse
+
+try:
+    from importlib.metadata import PathDistribution
+except ImportError:
+    from importlib_metadata import PathDistribution
+
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
+
+
+def normalize_name(name):
+    """https://www.python.org/dev/peps/pep-0503/#normalized-names"""
+    return re.sub(r'[-_.]+', '-', name).lower()
+
+
+def legacy_normalize_name(name):
+    """Like pkg_resources Distribution.key property"""
+    return re.sub(r'[-_]+', '-', name).lower()
+
+
+class Requirement(Requirement_):
+    def __init__(self, requirement_string):
+        super(Requirement, self).__init__(requirement_string)
+        self.normalized_name = normalize_name(self.name)
+        self.legacy_normalized_name = legacy_normalize_name(self.name)
+
+
+class Distribution(PathDistribution):
+    def __init__(self, path):
+        super(Distribution, self).__init__(Path(path))
+        self.name = self.metadata['Name']
+        self.normalized_name = normalize_name(self.name)
+        self.legacy_normalized_name = legacy_normalize_name(self.name)
+        self.requirements = [Requirement(r) for r in self.requires or []]
+        self.extras = [
+            v for k, v in self.metadata.items() if k == 'Provides-Extra']
+        self.py_version = self._parse_py_version(path)
+
+    def _parse_py_version(self, path):
+        # Try to parse the Python version from the path the metadata
+        # resides at (e.g. /usr/lib/pythonX.Y/site-packages/...)
+        res = re.search(r"/python(?P<pyver>\d+\.\d+)/", path)
+        if res:
+            return res.group('pyver')
+        # If that hasn't worked, attempt to parse it from the metadata
+        # directory name
+        res = re.search(r"-py(?P<pyver>\d+.\d+)[.-]egg-info$", path)
+        if res:
+            return res.group('pyver')
+        return None
+
+    def requirements_for_extra(self, extra):
+        extra_deps = []
+        for req in self.requirements:
+            if not req.marker:
+                continue
+            if req.marker.evaluate(get_marker_env(self, extra)):
+                extra_deps.append(req)
+        return extra_deps
+
+    def __repr__(self):
+        return '{} from {}'.format(self.name, self._path)
+
 
 class RpmVersion():
     def __init__(self, version_id):
-        version = parse_version(version_id)
+        version = parse(version_id)
         if isinstance(version._version, str):
             self.version = version._version
         else:
@@ -144,10 +208,20 @@ def convert(name, operator, version_id):
                            format(version_id, name)) from exc
 
 
-def normalize_name(name):
+def get_marker_env(dist, extra):
-    """https://www.python.org/dev/peps/pep-0503/#normalized-names"""
+    # packaging uses a default environment using
-    import re
+    # platform.python_version to evaluate if a dependency is relevant
-    return re.sub(r'[-_.]+', '-', name).lower()
+    # based on environment markers [1],
+    # e.g. requirement `argparse;python_version<"2.7"`
+    #
+    # Since we're running this script on one Python version while
+    # possibly evaluating packages for different versions, we
+    # set up an environment with the version we want to evaluate.
+    #
+    # [1] https://www.python.org/dev/peps/pep-0508/#environment-markers
+    return {"python_full_version": dist.py_version,
+            "python_version": dist.py_version,
+            "extra": extra}
 
 
 if __name__ == "__main__":
@@ -243,52 +317,21 @@ if __name__ == "__main__":
         if lower.endswith('.egg') or \
                 lower.endswith('.egg-info') or \
                 lower.endswith('.dist-info'):
-            # This import is very slow, so only do it if needed
+            dist = Distribution(f)
-            # - Notes from an attempted rewrite from pkg_resources to
-            #   importlib.metadata in 2020 can be found in the message of
-            #   the commit that added this line.
-            from pkg_resources import Distribution, FileMetadata, PathMetadata, Requirement, parse_version
-            dist_name = basename(f)
-            if isdir(f):
-                path_item = dirname(f)
-                metadata = PathMetadata(path_item, f)
-            else:
-                path_item = f
-                metadata = FileMetadata(f)
-            dist = Distribution.from_location(path_item, dist_name, metadata)
-            # Check if py_version is defined in the metadata file/directory name
             if not dist.py_version:
-                # Try to parse the Python version from the path the metadata
+                warn("Version for {!r} has not been found".format(dist), RuntimeWarning)
-                # resides at (e.g. /usr/lib/pythonX.Y/site-packages/...)
+                continue
-                import re
-                res = re.search(r"/python(?P<pyver>\d+\.\d+)/", path_item)
-                if res:
-                    dist.py_version = res.group('pyver')
-                else:
-                    warn("Version for {!r} has not been found".format(dist), RuntimeWarning)
-                    continue
 
-            # pkg_resources use platform.python_version to evaluate if a
+            # If processing an extras subpackage:
-            # dependency is relevant based on environment markers [1],
+            #   Check that the extras name is declared in the metadata, or
-            # e.g. requirement `argparse;python_version<"2.7"`
+            #   that there are some dependencies associated with the extras
-            #
+            #   name in the requires.txt (this is an outdated way to declare
-            # Since we're running this script on one Python version while
+            #   extras packages).
-            # possibly evaluating packages for different versions, we mock the
+            # - If there is an extras package declared only in requires.txt
-            # platform.python_version function. Discussed upstream [2].
+            #   without any dependencies, this check will fail. In that case
-            #
+            #   make sure to use updated metadata and declare the extras
-            # [1] https://www.python.org/dev/peps/pep-0508/#environment-markers
+            #   package there.
-            # [2] https://github.com/pypa/setuptools/pull/1275
+            if extras_subpackage and extras_subpackage not in dist.extras and not dist.requirements_for_extra(extras_subpackage):
-            import platform
-            platform.python_version = lambda: dist.py_version
-            platform.python_version_tuple = lambda: tuple(dist.py_version.split('.'))
-
-            # This is the PEP 503 normalized name.
-            # It does also convert dots to dashes, unlike dist.key.
-            # See https://bugzilla.redhat.com/show_bug.cgi?id=1791530
-            normalized_name = normalize_name(dist.project_name)
-
-            # If we're processing an extras subpackage, check that the extras exists
-            if extras_subpackage and extras_subpackage not in dist.extras:
                 print("*** PYTHON_EXTRAS_NOT_FOUND_ERROR___SEE_STDERR ***")
                 print(f"\nError: The package name contains an extras name `{extras_subpackage}` that was not found in the metadata.\n"
                       "Check if the extras were removed from the project. If so, consider removing the subpackage and obsoleting it from another.\n", file=stderr)
@@ -301,32 +344,32 @@ if __name__ == "__main__":
             if args.provides:
                 extras_suffix = f"[{extras_subpackage}]" if extras_subpackage else ""
                 # If egg/dist metadata says package name is python, we provide python(abi)
-                if dist.key == 'python':
+                if dist.normalized_name == 'python':
                     name = 'python(abi)'
                     if name not in py_deps:
                         py_deps[name] = []
                     py_deps[name].append(('==', dist.py_version))
                 if not args.legacy or not args.majorver_only:
                     if normalized_names_provide_legacy:
-                        name = 'python{}dist({}{})'.format(dist.py_version, dist.key, extras_suffix)
+                        name = 'python{}dist({}{})'.format(dist.py_version, dist.legacy_normalized_name, extras_suffix)
                         if name not in py_deps:
                             py_deps[name] = []
                     if normalized_names_provide_pep503:
-                        name_ = 'python{}dist({}{})'.format(dist.py_version, normalized_name, extras_suffix)
+                        name_ = 'python{}dist({}{})'.format(dist.py_version, dist.normalized_name, extras_suffix)
                         if name_ not in py_deps:
                             py_deps[name_] = []
                 if args.majorver_provides or args.majorver_only or \
                         (args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions):
                     if normalized_names_provide_legacy:
-                        pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.key, extras_suffix)
+                        pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.legacy_normalized_name, extras_suffix)
                         if pymajor_name not in py_deps:
                             py_deps[pymajor_name] = []
                     if normalized_names_provide_pep503:
-                        pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, normalized_name, extras_suffix)
+                        pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, dist.normalized_name, extras_suffix)
                         if pymajor_name_ not in py_deps:
                             py_deps[pymajor_name_] = []
                 if args.legacy or args.legacy_provides:
-                    legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.key)
+                    legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.legacy_normalized_name)
                     if legacy_name not in py_deps:
                         py_deps[legacy_name] = []
                 if dist.version:
@@ -351,7 +394,7 @@ if __name__ == "__main__":
             if args.requires or (args.recommends and dist.extras):
                 name = 'python(abi)'
                 # If egg/dist metadata says package name is python, we don't add dependency on python(abi)
-                if dist.key == 'python':
+                if dist.normalized_name == 'python':
                     py_abi = False
                     if name in py_deps:
                         py_deps.pop(name)
@@ -361,24 +404,21 @@ if __name__ == "__main__":
                     spec = ('==', dist.py_version)
                     if spec not in py_deps[name]:
                         py_deps[name].append(spec)
-                deps = dist.requires()
+
-                if args.recommends:
+                if extras_subpackage:
-                    depsextras = dist.requires(extras=dist.extras)
+                    deps = [d for d in dist.requirements_for_extra(extras_subpackage)]
-                    if not args.requires:
+                else:
-                        for dep in reversed(depsextras):
+                    deps = dist.requirements
-                            if dep in deps:
+
-                                depsextras.remove(dep)
-                    deps = depsextras
-                elif extras_subpackage:
-                    # Extras requires also contain the base requires included
-                    deps = [d for d in dist.requires(extras=[extras_subpackage]) if d not in dist.requires()]
                 # console_scripts/gui_scripts entry points need pkg_resources from setuptools
-                if ((dist.get_entry_map('console_scripts') or
+                if (dist.entry_points and
-                    dist.get_entry_map('gui_scripts')) and
                     (lower.endswith('.egg') or
                      lower.endswith('.egg-info'))):
-                    # stick them first so any more specific requirement overrides it
+                    groups = {ep.group for ep in dist.entry_points}
-                    deps.insert(0, Requirement.parse('setuptools'))
+                    if {"console_scripts", "gui_scripts"} & groups:
+                        # stick them first so any more specific requirement
+                        # overrides it
+                        deps.insert(0, Requirement('setuptools'))
                 # add requires/recommends based on egg/dist metadata
                 for dep in deps:
                     # Even if we're requiring `foo[bar]`, also require `foo`
@@ -389,67 +429,62 @@ if __name__ == "__main__":
                         # A dependency can have more than one extras,
                         # i.e. foo[bar,baz], so let's go through all of them
                         extras_suffixes += [f"[{e}]" for e in dep.extras]
+
                     for extras_suffix in extras_suffixes:
                         if normalized_names_require_pep503:
-                            dep_normalized_name = normalize_name(dep.project_name)
+                            dep_normalized_name = dep.normalized_name
                         else:
-                            dep_normalized_name = dep.key
+                            dep_normalized_name = dep.legacy_normalized_name
 
                         if args.legacy:
-                            name = 'pythonegg({})({})'.format(pyver_major, dep.key)
+                            name = 'pythonegg({})({})'.format(pyver_major, dep.legacy_normalized_name)
                         else:
                             if args.majorver_only:
                                 name = 'python{}dist({}{})'.format(pyver_major, dep_normalized_name, extras_suffix)
                             else:
                                 name = 'python{}dist({}{})'.format(dist.py_version, dep_normalized_name, extras_suffix)
-                        for spec in dep.specs:
+
-                            if name not in py_deps:
+                        if dep.marker and not args.recommends and not extras_subpackage:
-                                py_deps[name] = []
+                            if not dep.marker.evaluate(get_marker_env(dist, '')):
-                            if spec not in py_deps[name]:
+                                continue
-                                py_deps[name].append(spec)
+
-                        if not dep.specs:
+                        if name not in py_deps:
                             py_deps[name] = []
+                        for spec in dep.specifier:
+                            if (spec.operator, spec.version) not in py_deps[name]:
+                                py_deps[name].append((spec.operator, spec.version))
+
             # Unused, for automatic sub-package generation based on 'extras' from egg/dist metadata
             # TODO: implement in rpm later, or...?
             if args.extras:
-                deps = dist.requires()
+                print(dist.extras)
-                extras = dist.extras
+                for extra in dist.extras:
-                print(extras)
-                for extra in extras:
                     print('%%package\textras-{}'.format(extra))
-                    print('Summary:\t{} extra for {} python package'.format(extra, dist.key))
+                    print('Summary:\t{} extra for {} python package'.format(extra, dist.legacy_normalized_name))
                     print('Group:\t\tDevelopment/Python')
-                    depsextras = dist.requires(extras=[extra])
+                    for dep in dist.requirements_for_extra(extra):
-                    for dep in reversed(depsextras):
+                        for spec in dep.specifier:
-                        if dep in deps:
+                            if spec.operator == '!=':
-                            depsextras.remove(dep)
+                                print('Conflicts:\t{} {} {}'.format(dep.legacy_normalized_name, '==', spec.version))
-                    deps = depsextras
-                    for dep in deps:
-                        for spec in dep.specs:
-                            if spec[0] == '!=':
-                                print('Conflicts:\t{} {} {}'.format(dep.key, '==', spec[1]))
                             else:
-                                print('Requires:\t{} {} {}'.format(dep.key, spec[0], spec[1]))
+                                print('Requires:\t{} {} {}'.format(dep.legacy_normalized_name, spec.operator, spec.version))
                     print('%%description\t{}'.format(extra))
-                    print('{} extra for {} python package'.format(extra, dist.key))
+                    print('{} extra for {} python package'.format(extra, dist.legacy_normalized_name))
                     print('%%files\t\textras-{}\n'.format(extra))
             if args.conflicts:
                 # Should we really add conflicts for extras?
                 # Creating a meta package per extra with recommends on, which has
                 # the requires/conflicts in stead might be a better solution...
-                for dep in dist.requires(extras=dist.extras):
+                for dep in dist.requirements:
-                    name = dep.key
+                    for spec in dep.specifier:
-                    for spec in dep.specs:
+                        if spec.operator == '!=':
-                        if spec[0] == '!=':
+                            if dep.legacy_normalized_name not in py_deps:
-                            if name not in py_deps:
+                                py_deps[dep.legacy_normalized_name] = []
-                                py_deps[name] = []
+                            spec = ('==', spec.version)
-                            spec = ('==', spec[1])
+                            if spec not in py_deps[dep.legacy_normalized_name]:
-                            if spec not in py_deps[name]:
+                                py_deps[dep.legacy_normalized_name].append(spec)
-                                py_deps[name].append(spec)
 
-    names = list(py_deps.keys())
+    for name in sorted(py_deps):
-    names.sort()
-    for name in names:
         if py_deps[name]:
             # Print out versioned provides, requires, recommends, conflicts
             spec_list = []

tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt

@@ -1,4 +1,3 @@
-# Taken from pyreq2rpm, removed tests that are expected to fail
 foobar0~=2.4.8
 foobar1~=2.4.8.0
 foobar2~=2.4.8.1
@@ -91,10 +90,8 @@ pyparsing0
 pyparsing1>=2.0.1,!=2.0.4,!=2.1.2,!=2.1.6
 babel>=1.3,!=2.0
 
-# Tests for breakages in Fedora
 fedora-python-nb2plots==0+unknown
 
-# Other tests
 hugo1==1.0.0.dev7
 hugo2<=8a4
 hugo3!=11.1.1b14

tests/data/scripts_pythondistdeps/test-data.yaml

@@ -1216,6 +1216,15 @@
                 python3dist(backports-range) = 3.7.2
                 python3dist(backports.range) = 3.7.2
             requires: python(abi) = 3.7
+--requires --normalized-names-format pep503 --package-name python3-setuptools+certs:
+    --provides --majorver-provides --normalized-names-format pep503 --package-name python3-setuptools+certs:
+        usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info:
+            provides: |-
+                python3.9dist(setuptools[certs]) = 41.6
+                python3dist(setuptools[certs]) = 41.6
+            requires: |-
+                python(abi) = 3.9
+                python3.9dist(certifi) = 2016.9.26
 --requires --normalized-names-format pep503 --package-name python3-zope-component+testing:
     --provides --majorver-provides --normalized-names-format pep503 --package-name python3-zope-component+testing:
         usr/lib/python3.9/site-packages/zope.component-4.3.0-py3.9.egg-info: