rpms
/
python-rpm-generators
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

scripts/pythondistdeps: Backport switch to importlib.metadata from upstream 

Upstream change to importlib.metadata: https://github.com/rpm-software-management/rpm/pull/1317

Due to extras packages being hadled slightly differently by importlib,
one test case for this was added.  And due to changes in handling
requires.txt files, comments were removed from the pyreq2rpm.tests
testing package.

Also because of the switch, we removed the dependency on setuptools and
added a dependency on packaging.

Note: Some packages with egg-info files might provide a different name
due to this change if there is a conflict between the filename and the
name in the metadata. Previously, the filename was sometimes used to
parse the name, now it is always the content of that file, which is what
packaging does, and thus also pip and other Python tooling. Currently,
this is known to affect only 1 package in Fedora (ntpsec).

The resulting script is different from upstream because of not yet upstreamed changes in Fedora:
- scripts/pythondistdeps: Rework error messages
- scripts/pythondistdeps: Add parameter --package-name
- scripts/pythondistdeps: Implement provides/requires for extras packages
- pythondistdeps.py: When parsing extras name, take the rightmost +

These changes are proposed in this upstream PR: https://github.com/rpm-software-management/rpm/pull/1546
This commit is contained in:
Tomas Orsava 2021-02-17 12:18:26 +01:00
parent 2d631762c5
commit 438d8d3b70
4 changed files with 165 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
python-rpm-generators.spec
View File

@ -1,7 +1,7 @@
Name: python-rpm-generators
Summary: Dependency generators for Python RPMs
Version: 12
Release: 1%{?dist}
Release: 2%{?dist}
# Originally all those files were part of RPM, so license is kept here
License: GPLv2+
@ -21,7 +21,7 @@ BuildArch: noarch
%package -n python3-rpm-generators
Summary: %{summary}
Requires: python3-setuptools
Requires: python3-packaging
# We have parametric macro generators, we need RPM 4.16 (4.15.90+ is 4.16 alpha)
Requires: rpm > 4.15.90-0
# This contains the Lua functions we use:
@ -47,6 +47,11 @@ install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} *.py
%{_rpmconfigdir}/pythonbundles.py
%changelog
* Wed Feb 17 2021 Tomas Orsava <torsava@redhat.com> - 12-2
- scripts/pythondistdeps: Switch from using pkg_resources to importlib.metadata
for reading the egg/dist-info metadata
- The script no longer requires setuptools but instead requires packaging
* Wed Feb 03 2021 Miro Hrončok <mhroncok@redhat.com> - 12-1
- Disable the dist generators for Python 2
- https://fedoraproject.org/wiki/Changes/Disable_Python_2_Dist_RPM_Generators_and_Freeze_Python_2_Macros

263
pythondistdeps.py
View File

@ -11,21 +11,85 @@
# RPM python dependency generator, using .egg-info/.egg-link/.dist-info data
#
# Please know:
# - Notes from an attempted rewrite from pkg_resources to importlib.metadata in
# 2020 can be found in the message of the commit that added this line.
from __future__ import print_function
import argparse
from os.path import basename, dirname, isdir, sep
from sys import argv, stdin, stderr, version
from distutils.sysconfig import get_python_lib
from os.path import dirname, sep
import re
from sys import argv, stdin, stderr
from warnings import warn
from packaging.requirements import Requirement as Requirement_
from packaging.version import parse
try:
from importlib.metadata import PathDistribution
except ImportError:
from importlib_metadata import PathDistribution
try:
from pathlib import Path
except ImportError:
from pathlib2 import Path
def normalize_name(name):
"""https://www.python.org/dev/peps/pep-0503/#normalized-names"""
return re.sub(r'[-_.]+', '-', name).lower()
def legacy_normalize_name(name):
"""Like pkg_resources Distribution.key property"""
return re.sub(r'[-_]+', '-', name).lower()
class Requirement(Requirement_):
def __init__(self, requirement_string):
super(Requirement, self).__init__(requirement_string)
self.normalized_name = normalize_name(self.name)
self.legacy_normalized_name = legacy_normalize_name(self.name)
class Distribution(PathDistribution):
def __init__(self, path):
super(Distribution, self).__init__(Path(path))
self.name = self.metadata['Name']
self.normalized_name = normalize_name(self.name)
self.legacy_normalized_name = legacy_normalize_name(self.name)
self.requirements = [Requirement(r) for r in self.requires or []]
self.extras = [
v for k, v in self.metadata.items() if k == 'Provides-Extra']
self.py_version = self._parse_py_version(path)
def _parse_py_version(self, path):
# Try to parse the Python version from the path the metadata
# resides at (e.g. /usr/lib/pythonX.Y/site-packages/...)
res = re.search(r"/python(?P<pyver>\d+\.\d+)/", path)
if res:
return res.group('pyver')
# If that hasn't worked, attempt to parse it from the metadata
# directory name
res = re.search(r"-py(?P<pyver>\d+.\d+)[.-]egg-info$", path)
if res:
return res.group('pyver')
return None
def requirements_for_extra(self, extra):
extra_deps = []
for req in self.requirements:
if not req.marker:
continue
if req.marker.evaluate(get_marker_env(self, extra)):
extra_deps.append(req)
return extra_deps
def __repr__(self):
return '{} from {}'.format(self.name, self._path)
class RpmVersion():
def __init__(self, version_id):
version = parse_version(version_id)
version = parse(version_id)
if isinstance(version._version, str):
self.version = version._version
else:
@ -144,10 +208,20 @@ def convert(name, operator, version_id):
format(version_id, name)) from exc
def normalize_name(name):
"""https://www.python.org/dev/peps/pep-0503/#normalized-names"""
import re
return re.sub(r'[-_.]+', '-', name).lower()
def get_marker_env(dist, extra):
# packaging uses a default environment using
# platform.python_version to evaluate if a dependency is relevant
# based on environment markers [1],
# e.g. requirement `argparse;python_version<"2.7"`
#
# Since we're running this script on one Python version while
# possibly evaluating packages for different versions, we
# set up an environment with the version we want to evaluate.
#
# [1] https://www.python.org/dev/peps/pep-0508/#environment-markers
return {"python_full_version": dist.py_version,
"python_version": dist.py_version,
"extra": extra}
if __name__ == "__main__":
@ -243,52 +317,21 @@ if __name__ == "__main__":
if lower.endswith('.egg') or \
lower.endswith('.egg-info') or \
lower.endswith('.dist-info'):
# This import is very slow, so only do it if needed
# - Notes from an attempted rewrite from pkg_resources to
# importlib.metadata in 2020 can be found in the message of
# the commit that added this line.
from pkg_resources import Distribution, FileMetadata, PathMetadata, Requirement, parse_version
dist_name = basename(f)
if isdir(f):
path_item = dirname(f)
metadata = PathMetadata(path_item, f)
else:
path_item = f
metadata = FileMetadata(f)
dist = Distribution.from_location(path_item, dist_name, metadata)
# Check if py_version is defined in the metadata file/directory name
dist = Distribution(f)
if not dist.py_version:
# Try to parse the Python version from the path the metadata
# resides at (e.g. /usr/lib/pythonX.Y/site-packages/...)
import re
res = re.search(r"/python(?P<pyver>\d+\.\d+)/", path_item)
if res:
dist.py_version = res.group('pyver')
else:
warn("Version for {!r} has not been found".format(dist), RuntimeWarning)
continue
warn("Version for {!r} has not been found".format(dist), RuntimeWarning)
continue
# pkg_resources use platform.python_version to evaluate if a
# dependency is relevant based on environment markers [1],
# e.g. requirement `argparse;python_version<"2.7"`
#
# Since we're running this script on one Python version while
# possibly evaluating packages for different versions, we mock the
# platform.python_version function. Discussed upstream [2].
#
# [1] https://www.python.org/dev/peps/pep-0508/#environment-markers
# [2] https://github.com/pypa/setuptools/pull/1275
import platform
platform.python_version = lambda: dist.py_version
platform.python_version_tuple = lambda: tuple(dist.py_version.split('.'))
# This is the PEP 503 normalized name.
# It does also convert dots to dashes, unlike dist.key.
# See https://bugzilla.redhat.com/show_bug.cgi?id=1791530
normalized_name = normalize_name(dist.project_name)
# If we're processing an extras subpackage, check that the extras exists
if extras_subpackage and extras_subpackage not in dist.extras:
# If processing an extras subpackage:
# Check that the extras name is declared in the metadata, or
# that there are some dependencies associated with the extras
# name in the requires.txt (this is an outdated way to declare
# extras packages).
# - If there is an extras package declared only in requires.txt
# without any dependencies, this check will fail. In that case
# make sure to use updated metadata and declare the extras
# package there.
if extras_subpackage and extras_subpackage not in dist.extras and not dist.requirements_for_extra(extras_subpackage):
print("*** PYTHON_EXTRAS_NOT_FOUND_ERROR___SEE_STDERR ***")
print(f"\nError: The package name contains an extras name `{extras_subpackage}` that was not found in the metadata.\n"
"Check if the extras were removed from the project. If so, consider removing the subpackage and obsoleting it from another.\n", file=stderr)
@ -301,32 +344,32 @@ if __name__ == "__main__":
if args.provides:
extras_suffix = f"[{extras_subpackage}]" if extras_subpackage else ""
# If egg/dist metadata says package name is python, we provide python(abi)
if dist.key == 'python':
if dist.normalized_name == 'python':
name = 'python(abi)'
if name not in py_deps:
py_deps[name] = []
py_deps[name].append(('==', dist.py_version))
if not args.legacy or not args.majorver_only:
if normalized_names_provide_legacy:
name = 'python{}dist({}{})'.format(dist.py_version, dist.key, extras_suffix)
name = 'python{}dist({}{})'.format(dist.py_version, dist.legacy_normalized_name, extras_suffix)
if name not in py_deps:
py_deps[name] = []
if normalized_names_provide_pep503:
name_ = 'python{}dist({}{})'.format(dist.py_version, normalized_name, extras_suffix)
name_ = 'python{}dist({}{})'.format(dist.py_version, dist.normalized_name, extras_suffix)
if name_ not in py_deps:
py_deps[name_] = []
if args.majorver_provides or args.majorver_only or \
(args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions):
if normalized_names_provide_legacy:
pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.key, extras_suffix)
pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.legacy_normalized_name, extras_suffix)
if pymajor_name not in py_deps:
py_deps[pymajor_name] = []
if normalized_names_provide_pep503:
pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, normalized_name, extras_suffix)
pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, dist.normalized_name, extras_suffix)
if pymajor_name_ not in py_deps:
py_deps[pymajor_name_] = []
if args.legacy or args.legacy_provides:
legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.key)
legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.legacy_normalized_name)
if legacy_name not in py_deps:
py_deps[legacy_name] = []
if dist.version:
@ -351,7 +394,7 @@ if __name__ == "__main__":
if args.requires or (args.recommends and dist.extras):
name = 'python(abi)'
# If egg/dist metadata says package name is python, we don't add dependency on python(abi)
if dist.key == 'python':
if dist.normalized_name == 'python':
py_abi = False
if name in py_deps:
py_deps.pop(name)
@ -361,24 +404,21 @@ if __name__ == "__main__":
spec = ('==', dist.py_version)
if spec not in py_deps[name]:
py_deps[name].append(spec)
deps = dist.requires()
if args.recommends:
depsextras = dist.requires(extras=dist.extras)
if not args.requires:
for dep in reversed(depsextras):
if dep in deps:
depsextras.remove(dep)
deps = depsextras
elif extras_subpackage:
# Extras requires also contain the base requires included
deps = [d for d in dist.requires(extras=[extras_subpackage]) if d not in dist.requires()]
if extras_subpackage:
deps = [d for d in dist.requirements_for_extra(extras_subpackage)]
else:
deps = dist.requirements
# console_scripts/gui_scripts entry points need pkg_resources from setuptools
if ((dist.get_entry_map('console_scripts') or
dist.get_entry_map('gui_scripts')) and
if (dist.entry_points and
(lower.endswith('.egg') or
lower.endswith('.egg-info'))):
# stick them first so any more specific requirement overrides it
deps.insert(0, Requirement.parse('setuptools'))
groups = {ep.group for ep in dist.entry_points}
if {"console_scripts", "gui_scripts"} & groups:
# stick them first so any more specific requirement
# overrides it
deps.insert(0, Requirement('setuptools'))
# add requires/recommends based on egg/dist metadata
for dep in deps:
# Even if we're requiring `foo[bar]`, also require `foo`
@ -389,67 +429,62 @@ if __name__ == "__main__":
# A dependency can have more than one extras,
# i.e. foo[bar,baz], so let's go through all of them
extras_suffixes += [f"[{e}]" for e in dep.extras]
for extras_suffix in extras_suffixes:
if normalized_names_require_pep503:
dep_normalized_name = normalize_name(dep.project_name)
dep_normalized_name = dep.normalized_name
else:
dep_normalized_name = dep.key
dep_normalized_name = dep.legacy_normalized_name
if args.legacy:
name = 'pythonegg({})({})'.format(pyver_major, dep.key)
name = 'pythonegg({})({})'.format(pyver_major, dep.legacy_normalized_name)
else:
if args.majorver_only:
name = 'python{}dist({}{})'.format(pyver_major, dep_normalized_name, extras_suffix)
else:
name = 'python{}dist({}{})'.format(dist.py_version, dep_normalized_name, extras_suffix)
for spec in dep.specs:
if name not in py_deps:
py_deps[name] = []
if spec not in py_deps[name]:
py_deps[name].append(spec)
if not dep.specs:
if dep.marker and not args.recommends and not extras_subpackage:
if not dep.marker.evaluate(get_marker_env(dist, '')):
continue
if name not in py_deps:
py_deps[name] = []
for spec in dep.specifier:
if (spec.operator, spec.version) not in py_deps[name]:
py_deps[name].append((spec.operator, spec.version))
# Unused, for automatic sub-package generation based on 'extras' from egg/dist metadata
# TODO: implement in rpm later, or...?
if args.extras:
deps = dist.requires()
extras = dist.extras
print(extras)
for extra in extras:
print(dist.extras)
for extra in dist.extras:
print('%%package\textras-{}'.format(extra))
print('Summary:\t{} extra for {} python package'.format(extra, dist.key))
print('Summary:\t{} extra for {} python package'.format(extra, dist.legacy_normalized_name))
print('Group:\t\tDevelopment/Python')
depsextras = dist.requires(extras=[extra])
for dep in reversed(depsextras):
if dep in deps:
depsextras.remove(dep)
deps = depsextras
for dep in deps:
for spec in dep.specs:
if spec[0] == '!=':
print('Conflicts:\t{} {} {}'.format(dep.key, '==', spec[1]))
for dep in dist.requirements_for_extra(extra):
for spec in dep.specifier:
if spec.operator == '!=':
print('Conflicts:\t{} {} {}'.format(dep.legacy_normalized_name, '==', spec.version))
else:
print('Requires:\t{} {} {}'.format(dep.key, spec[0], spec[1]))
print('Requires:\t{} {} {}'.format(dep.legacy_normalized_name, spec.operator, spec.version))
print('%%description\t{}'.format(extra))
print('{} extra for {} python package'.format(extra, dist.key))
print('{} extra for {} python package'.format(extra, dist.legacy_normalized_name))
print('%%files\t\textras-{}\n'.format(extra))
if args.conflicts:
# Should we really add conflicts for extras?
# Creating a meta package per extra with recommends on, which has
# the requires/conflicts in stead might be a better solution...
for dep in dist.requires(extras=dist.extras):
name = dep.key
for spec in dep.specs:
if spec[0] == '!=':
if name not in py_deps:
py_deps[name] = []
spec = ('==', spec[1])
if spec not in py_deps[name]:
py_deps[name].append(spec)
for dep in dist.requirements:
for spec in dep.specifier:
if spec.operator == '!=':
if dep.legacy_normalized_name not in py_deps:
py_deps[dep.legacy_normalized_name] = []
spec = ('==', spec.version)
if spec not in py_deps[dep.legacy_normalized_name]:
py_deps[dep.legacy_normalized_name].append(spec)
names = list(py_deps.keys())
names.sort()
for name in names:
for name in sorted(py_deps):
if py_deps[name]:
# Print out versioned provides, requires, recommends, conflicts
spec_list = []

3
tests/data/scripts_pythondistdeps/pyreq2rpm.tests-2020.04.07.024dab0-py3.9.egg-info/requires.txt
View File

@ -1,4 +1,3 @@
# Taken from pyreq2rpm, removed tests that are expected to fail
foobar0~=2.4.8
foobar1~=2.4.8.0
foobar2~=2.4.8.1
@ -91,10 +90,8 @@ pyparsing0
pyparsing1>=2.0.1,!=2.0.4,!=2.1.2,!=2.1.6
babel>=1.3,!=2.0
# Tests for breakages in Fedora
fedora-python-nb2plots==0+unknown
# Other tests
hugo1==1.0.0.dev7
hugo2<=8a4
hugo3!=11.1.1b14

9
tests/data/scripts_pythondistdeps/test-data.yaml
View File

@ -1216,6 +1216,15 @@
python3dist(backports-range) = 3.7.2
python3dist(backports.range) = 3.7.2
requires: python(abi) = 3.7
--requires --normalized-names-format pep503 --package-name python3-setuptools+certs:
--provides --majorver-provides --normalized-names-format pep503 --package-name python3-setuptools+certs:
usr/lib/python3.9/site-packages/setuptools-41.6.0.dist-info:
provides: |-
python3.9dist(setuptools[certs]) = 41.6
python3dist(setuptools[certs]) = 41.6
requires: |-
python(abi) = 3.9
python3.9dist(certifi) = 2016.9.26
--requires --normalized-names-format pep503 --package-name python3-zope-component+testing:
--provides --majorver-provides --normalized-names-format pep503 --package-name python3-zope-component+testing:
usr/lib/python3.9/site-packages/zope.component-4.3.0-py3.9.egg-info: