python-rpm-generators/pythondistdeps.py

458 lines
23 KiB
Python
Raw Normal View History

#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
# Copyright 2010 Per Øyvind Karlsen <proyvind@moondrake.org>
# Copyright 2015 Neal Gompa <ngompa13@gmail.com>
2020-04-30 13:57:15 +00:00
# Copyright 2020 SUSE LLC
#
# This program is free software. It may be redistributed and/or modified under
# the terms of the LGPL version 2.1 (or later).
#
# RPM python dependency generator, using .egg-info/.egg-link/.dist-info data
#
scripts/pythondistdeps: Notes from an attempted rewrite to importlib.metadata Notes from an attempted rewrite from pkg_resources to importlib.metadata in 2020: 1. While pkg_resources can open a metadata on a specified path (Distribution.from_location()), importlib provides access only to "installed package metadata", i.e. the the dist-info or egg-info directory must be "discoverable", i.e. on the sys.path. - Thankfully only the dist/egg-info directory must exist, the corresponding Python module does not have to be present. - The problems this causes: (a) You have to manipulate the sys.path to add the specific location of the site-packages directory inside the buildroot (b) If you have package "foo" in this newly added directory on sys.path and there is some problem and its dist/egg-info metadata are not found, importlib.metadata continues searching the sys.path and may discover a package with the same name (possibly same version) outside the buildroot. To get around this, you can manipulate the sys.path to remove all other "site-packages" directories. But you have to leave the standard library there, because importlib may import other modules (in my testing: base64, quopri, random, socket, calendar, uu) (c) I have not tested how well it works if you're ispecting metadata of different Python versions than the one you run the script with (especially Python 2 vs Python 3). This might also cause problems with dependency specifiers (i.e. python_version != "3.4") 2. Handling of dependencies (requires) is problematic in importlib.metadata - pkg_resources provides a way to separately list standard requires and a requires for each "extras" category. importlib does not provide this, it only spits out a list of strings, each string in the format: - 'packaging>=14', - 'towncrier>=18.5.0; extra == "docs"', or - 'psutil<6,>=5.6.1; (python_version != "3.4") and extra == "testing" you can either parse these with a regex (fragile) or use the external `packaging` Python module. `packaging`, however, also doesn't have a great support for figuring out extra dependencies, it provides the marker api: - <Marker(\'python_version != "3.4" and extra == "testing"\')> you can use Marker api to evaluate the condition, but not to parse. For parsing you can access the private api Marker._markers: - marker._markers=[[(<Variable('python_version')>, <Op('!=')>, \ <Value('3.4')>)], 'and', (<Variable('extra')>, <Op('==')>, \ <Value('testing')>)] which beyond the problem of being private is also not very useful for parsing due to its structure. - pkg_resources also provides version parsing, which importlib does not and `packaging` needs to be used - importlib is part of the standard library, but packaging and its 2 runtime dependencies (pyparsing and six) are not, and therefore we would go from 1 dependency to 3 3. A few minor issues, more in the next section about equivalents. importlib.metadata.distribution equivalents of pkg_resources.Distribution attributes: - pkg_resources: dist.py_version importlib: # not implemented (but can be guessed from the /usr/lib/pythonXX.YY/ path) - pkg_resources: dist.project_name importlib: dist.metadata['name'] - pkg_resources: dist.key importlib: # not implemented - pkg_resources: dist.version importlib: dist.version - pkg_resources: dist.requires() importlib: dist.requires # but returns strings with almost no parsing done, and also lists extras - pkg_resources: dist.requires(extras=dist.extras) importlib: # not implemented, has to be parsed from dist.requires - pkg_resources: dist.get_entry_map('console_scripts') importlib: [ep for ep in importlib.metadata.entry_points()['console_scripts'] if ep.name == pkg][0] # I have not found a better way to get the console_scripts - pkg_resources: dist.get_entry_map('gui_scripts') importlib: # Presumably same as console_scripts, but untested
2020-04-08 16:12:09 +00:00
# Please know:
# - Notes from an attempted rewrite from pkg_resources to importlib.metadata in
# 2020 can be found in the message of the commit that added this line.
from __future__ import print_function
2020-04-30 13:57:15 +00:00
import argparse
from os.path import basename, dirname, isdir, sep
from sys import argv, stdin, stderr, version
from distutils.sysconfig import get_python_lib
from warnings import warn
2020-04-10 05:41:01 +00:00
class RpmVersion():
def __init__(self, version_id):
version = parse_version(version_id)
if isinstance(version._version, str):
self.version = version._version
else:
self.epoch = version._version.epoch
self.version = list(version._version.release)
self.pre = version._version.pre
self.dev = version._version.dev
self.post = version._version.post
def increment(self):
self.version[-1] += 1
self.pre = None
self.dev = None
self.post = None
return self
def __str__(self):
if isinstance(self.version, str):
return self.version
if self.epoch:
rpm_epoch = str(self.epoch) + ':'
else:
rpm_epoch = ''
while len(self.version) > 1 and self.version[-1] == 0:
2020-04-10 05:41:01 +00:00
self.version.pop()
rpm_version = '.'.join(str(x) for x in self.version)
if self.pre:
rpm_suffix = '~{}'.format(''.join(str(x) for x in self.pre))
elif self.dev:
rpm_suffix = '~~{}'.format(''.join(str(x) for x in self.dev))
2020-04-10 05:41:01 +00:00
elif self.post:
rpm_suffix = '^post{}'.format(self.post[1])
else:
rpm_suffix = ''
return '{}{}{}'.format(rpm_epoch, rpm_version, rpm_suffix)
2020-04-30 13:57:15 +00:00
2020-04-10 05:41:01 +00:00
def convert_compatible(name, operator, version_id):
if version_id.endswith('.*'):
print("*** INVALID_REQUIREMENT_ERROR___SEE_STDERR ***")
print('Invalid requirement: {} {} {}'.format(name, operator, version_id), file=stderr)
2020-04-30 13:57:15 +00:00
exit(65) # os.EX_DATAERR
2020-04-10 05:41:01 +00:00
version = RpmVersion(version_id)
if len(version.version) == 1:
print("*** INVALID_REQUIREMENT_ERROR___SEE_STDERR ***")
print('Invalid requirement: {} {} {}'.format(name, operator, version_id), file=stderr)
2020-04-30 13:57:15 +00:00
exit(65) # os.EX_DATAERR
2020-04-10 05:41:01 +00:00
upper_version = RpmVersion(version_id)
upper_version.version.pop()
upper_version.increment()
return '({} >= {} with {} < {})'.format(
name, version, name, upper_version)
2020-04-30 13:57:15 +00:00
2020-04-10 05:41:01 +00:00
def convert_equal(name, operator, version_id):
if version_id.endswith('.*'):
version_id = version_id[:-2] + '.0'
return convert_compatible(name, '~=', version_id)
version = RpmVersion(version_id)
return '{} = {}'.format(name, version)
2020-04-30 13:57:15 +00:00
2020-04-10 05:41:01 +00:00
def convert_arbitrary_equal(name, operator, version_id):
if version_id.endswith('.*'):
print("*** INVALID_REQUIREMENT_ERROR___SEE_STDERR ***")
print('Invalid requirement: {} {} {}'.format(name, operator, version_id), file=stderr)
2020-04-30 13:57:15 +00:00
exit(65) # os.EX_DATAERR
2020-04-10 05:41:01 +00:00
version = RpmVersion(version_id)
return '{} = {}'.format(name, version)
2020-04-30 13:57:15 +00:00
2020-04-10 05:41:01 +00:00
def convert_not_equal(name, operator, version_id):
if version_id.endswith('.*'):
version_id = version_id[:-2]
version = RpmVersion(version_id)
lower_version = RpmVersion(version_id).increment()
else:
version = RpmVersion(version_id)
lower_version = version
return '({} < {} or {} > {})'.format(
name, version, name, lower_version)
2020-04-30 13:57:15 +00:00
2020-04-10 05:41:01 +00:00
def convert_ordered(name, operator, version_id):
if version_id.endswith('.*'):
# PEP 440 does not define semantics for prefix matching
# with ordered comparisons
version_id = version_id[:-2]
version = RpmVersion(version_id)
if '>' == operator:
# distutils does not behave this way, but this is
# their recommendation
# https://mail.python.org/archives/list/distutils-sig@python.org/thread/NWEQVTCX5CR2RKW2LT4H77PJTEINSX7P/
operator = '>='
version.increment()
else:
version = RpmVersion(version_id)
return '{} {} {}'.format(name, operator, version)
2020-04-30 13:57:15 +00:00
2020-04-10 05:41:01 +00:00
OPERATORS = {'~=': convert_compatible,
'==': convert_equal,
'===': convert_arbitrary_equal,
'!=': convert_not_equal,
'<=': convert_ordered,
2020-04-30 13:57:15 +00:00
'<': convert_ordered,
2020-04-10 05:41:01 +00:00
'>=': convert_ordered,
2020-04-30 13:57:15 +00:00
'>': convert_ordered}
2020-04-10 05:41:01 +00:00
def convert(name, operator, version_id):
try:
return OPERATORS[operator](name, operator, version_id)
except Exception as exc:
raise RuntimeError("Cannot process Python package version `{}` for name `{}`".
format(version_id, name)) from exc
2020-04-10 05:41:01 +00:00
def normalize_name(name):
"""https://www.python.org/dev/peps/pep-0503/#normalized-names"""
import re
return re.sub(r'[-_.]+', '-', name).lower()
if __name__ == "__main__":
"""To allow this script to be importable (and its classes/functions
reused), actions are performed only when run as a main script."""
parser = argparse.ArgumentParser(prog=argv[0])
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-P', '--provides', action='store_true', help='Print Provides')
group.add_argument('-R', '--requires', action='store_true', help='Print Requires')
group.add_argument('-r', '--recommends', action='store_true', help='Print Recommends')
group.add_argument('-C', '--conflicts', action='store_true', help='Print Conflicts')
group.add_argument('-E', '--extras', action='store_true', help='[Unused] Generate spec file snippets for extras subpackages')
group_majorver = parser.add_mutually_exclusive_group()
group_majorver.add_argument('-M', '--majorver-provides', action='store_true', help='Print extra Provides with Python major version only')
group_majorver.add_argument('--majorver-provides-versions', action='append',
help='Print extra Provides with Python major version only for listed '
'Python VERSIONS (appended or comma separated without spaces, e.g. 2.7,3.9)')
parser.add_argument('-m', '--majorver-only', action='store_true', help='Print Provides/Requires with Python major version only')
parser.add_argument('-n', '--normalized-names-format', action='store',
default="legacy-dots", choices=["pep503", "legacy-dots"],
help='Format of normalized names according to pep503 or legacy format that allows dots [default]')
parser.add_argument('--normalized-names-provide-both', action='store_true',
help='Provide both `pep503` and `legacy-dots` format of normalized names (useful for a transition period)')
parser.add_argument('-L', '--legacy-provides', action='store_true', help='Print extra legacy pythonegg Provides')
parser.add_argument('-l', '--legacy', action='store_true', help='Print legacy pythonegg Provides/Requires instead')
parser.add_argument('--require-extras-subpackages', action='store_true',
help="If there is a dependency on a package with extras functionality, require the extras subpackage")
parser.add_argument('--package-name', action='store', help="Name of the RPM package that's being inspected. Required for extras requires/provides to work.")
parser.add_argument('files', nargs=argparse.REMAINDER, help="Files from the RPM package that are to be inspected, can also be supplied on stdin")
args = parser.parse_args()
py_abi = args.requires
py_deps = {}
if args.majorver_provides_versions:
# Go through the arguments (can be specified multiple times),
# and parse individual versions (can be comma-separated)
args.majorver_provides_versions = [v for vstring in args.majorver_provides_versions
for v in vstring.split(",")]
# If normalized_names_require_pep503 is True we require the pep503
# normalized name, if it is False we provide the legacy normalized name
normalized_names_require_pep503 = args.normalized_names_format == "pep503"
# If normalized_names_provide_pep503/legacy is True we provide the
# pep503/legacy normalized name, if it is False we don't
normalized_names_provide_pep503 = \
args.normalized_names_format == "pep503" or args.normalized_names_provide_both
normalized_names_provide_legacy = \
args.normalized_names_format == "legacy-dots" or args.normalized_names_provide_both
# At least one type of normalization must be provided
assert normalized_names_provide_pep503 or normalized_names_provide_legacy
# Is this script being run for an extras subpackage?
extras_subpackage = None
if args.package_name:
package_name_parts = args.package_name.partition('+')
extras_subpackage = package_name_parts[2] or None
for f in (args.files or stdin.readlines()):
f = f.strip()
lower = f.lower()
name = 'python(abi)'
# add dependency based on path, versioned if within versioned python directory
if py_abi and (lower.endswith('.py') or lower.endswith('.pyc') or lower.endswith('.pyo')):
if name not in py_deps:
py_deps[name] = []
purelib = get_python_lib(standard_lib=0, plat_specific=0).split(version[:3])[0]
platlib = get_python_lib(standard_lib=0, plat_specific=1).split(version[:3])[0]
for lib in (purelib, platlib):
if lib in f:
spec = ('==', f.split(lib)[1].split(sep)[0])
if spec not in py_deps[name]:
py_deps[name].append(spec)
# XXX: hack to workaround RPM internal dependency generator not passing directories
lower_dir = dirname(lower)
if lower_dir.endswith('.egg') or \
lower_dir.endswith('.egg-info') or \
lower_dir.endswith('.dist-info'):
lower = lower_dir
f = dirname(f)
# Determine provide, requires, conflicts & recommends based on egg/dist metadata
if lower.endswith('.egg') or \
lower.endswith('.egg-info') or \
lower.endswith('.dist-info'):
# This import is very slow, so only do it if needed
# - Notes from an attempted rewrite from pkg_resources to
# importlib.metadata in 2020 can be found in the message of
# the commit that added this line.
from pkg_resources import Distribution, FileMetadata, PathMetadata, Requirement, parse_version
dist_name = basename(f)
if isdir(f):
path_item = dirname(f)
metadata = PathMetadata(path_item, f)
else:
path_item = f
metadata = FileMetadata(f)
dist = Distribution.from_location(path_item, dist_name, metadata)
# Check if py_version is defined in the metadata file/directory name
if not dist.py_version:
# Try to parse the Python version from the path the metadata
# resides at (e.g. /usr/lib/pythonX.Y/site-packages/...)
import re
res = re.search(r"/python(?P<pyver>\d+\.\d+)/", path_item)
if res:
dist.py_version = res.group('pyver')
else:
warn("Version for {!r} has not been found".format(dist), RuntimeWarning)
continue
# pkg_resources use platform.python_version to evaluate if a
# dependency is relevant based on environment markers [1],
# e.g. requirement `argparse;python_version<"2.7"`
#
# Since we're running this script on one Python version while
# possibly evaluating packages for different versions, we mock the
# platform.python_version function. Discussed upstream [2].
#
# [1] https://www.python.org/dev/peps/pep-0508/#environment-markers
# [2] https://github.com/pypa/setuptools/pull/1275
import platform
platform.python_version = lambda: dist.py_version
# This is the PEP 503 normalized name.
# It does also convert dots to dashes, unlike dist.key.
# See https://bugzilla.redhat.com/show_bug.cgi?id=1791530
normalized_name = normalize_name(dist.project_name)
# If we're processing an extras subpackage, check that the extras exists
if extras_subpackage and extras_subpackage not in dist.extras:
print("*** PYTHON_EXTRAS_NOT_FOUND_ERROR___SEE_STDERR ***")
print(f"\nError: The package name contains an extras name `{extras_subpackage}` that was not found in the metadata.\n"
"Check if the extras were removed from the project. If so, consider removing the subpackage and obsoleting it from another.\n", file=stderr)
exit(65) # os.EX_DATAERR
if args.majorver_provides or args.majorver_provides_versions or \
args.majorver_only or args.legacy_provides or args.legacy:
# Get the Python major version
pyver_major = dist.py_version.split('.')[0]
if args.provides:
extras_suffix = f"[{extras_subpackage}]" if extras_subpackage else ""
# If egg/dist metadata says package name is python, we provide python(abi)
if dist.key == 'python':
name = 'python(abi)'
if name not in py_deps:
py_deps[name] = []
py_deps[name].append(('==', dist.py_version))
if not args.legacy or not args.majorver_only:
if normalized_names_provide_legacy:
name = 'python{}dist({}{})'.format(dist.py_version, dist.key, extras_suffix)
if name not in py_deps:
py_deps[name] = []
if normalized_names_provide_pep503:
name_ = 'python{}dist({}{})'.format(dist.py_version, normalized_name, extras_suffix)
if name_ not in py_deps:
py_deps[name_] = []
if args.majorver_provides or args.majorver_only or \
(args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions):
if normalized_names_provide_legacy:
pymajor_name = 'python{}dist({}{})'.format(pyver_major, dist.key, extras_suffix)
if pymajor_name not in py_deps:
py_deps[pymajor_name] = []
if normalized_names_provide_pep503:
pymajor_name_ = 'python{}dist({}{})'.format(pyver_major, normalized_name, extras_suffix)
if pymajor_name_ not in py_deps:
py_deps[pymajor_name_] = []
if args.legacy or args.legacy_provides:
legacy_name = 'pythonegg({})({})'.format(pyver_major, dist.key)
if legacy_name not in py_deps:
py_deps[legacy_name] = []
if dist.version:
version = dist.version
spec = ('==', version)
if normalized_names_provide_legacy:
if spec not in py_deps[name]:
py_deps[name].append(spec)
if args.majorver_provides or \
(args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions):
py_deps[pymajor_name].append(spec)
if normalized_names_provide_pep503:
if spec not in py_deps[name_]:
py_deps[name_].append(spec)
if args.majorver_provides or \
(args.majorver_provides_versions and dist.py_version in args.majorver_provides_versions):
py_deps[pymajor_name_].append(spec)
if args.legacy or args.legacy_provides:
if spec not in py_deps[legacy_name]:
py_deps[legacy_name].append(spec)
if args.requires or (args.recommends and dist.extras):
name = 'python(abi)'
# If egg/dist metadata says package name is python, we don't add dependency on python(abi)
if dist.key == 'python':
py_abi = False
if name in py_deps:
py_deps.pop(name)
elif py_abi and dist.py_version:
if name not in py_deps:
py_deps[name] = []
spec = ('==', dist.py_version)
if spec not in py_deps[name]:
py_deps[name].append(spec)
deps = dist.requires()
if args.recommends:
depsextras = dist.requires(extras=dist.extras)
if not args.requires:
for dep in reversed(depsextras):
if dep in deps:
depsextras.remove(dep)
deps = depsextras
elif extras_subpackage:
# Extras requires also contain the base requires included
deps = [d for d in dist.requires(extras=[extras_subpackage]) if d not in dist.requires()]
# console_scripts/gui_scripts entry points need pkg_resources from setuptools
if ((dist.get_entry_map('console_scripts') or
dist.get_entry_map('gui_scripts')) and
(lower.endswith('.egg') or
lower.endswith('.egg-info'))):
# stick them first so any more specific requirement overrides it
deps.insert(0, Requirement.parse('setuptools'))
# add requires/recommends based on egg/dist metadata
for dep in deps:
# Even if we're requiring `foo[bar]`, also require `foo`
# to be safe, and to make it discoverable through
# `repoquery --whatrequires`
extras_suffixes = [""]
if args.require_extras_subpackages and dep.extras:
# A dependency can have more than one extras,
# i.e. foo[bar,baz], so let's go through all of them
extras_suffixes += [f"[{e}]" for e in dep.extras]
for extras_suffix in extras_suffixes:
if normalized_names_require_pep503:
dep_normalized_name = normalize_name(dep.project_name)
else:
dep_normalized_name = dep.key
if args.legacy:
name = 'pythonegg({})({})'.format(pyver_major, dep.key)
else:
if args.majorver_only:
name = 'python{}dist({}{})'.format(pyver_major, dep_normalized_name, extras_suffix)
else:
name = 'python{}dist({}{})'.format(dist.py_version, dep_normalized_name, extras_suffix)
for spec in dep.specs:
if name not in py_deps:
py_deps[name] = []
if spec not in py_deps[name]:
py_deps[name].append(spec)
if not dep.specs:
py_deps[name] = []
# Unused, for automatic sub-package generation based on 'extras' from egg/dist metadata
# TODO: implement in rpm later, or...?
if args.extras:
deps = dist.requires()
extras = dist.extras
print(extras)
for extra in extras:
print('%%package\textras-{}'.format(extra))
print('Summary:\t{} extra for {} python package'.format(extra, dist.key))
print('Group:\t\tDevelopment/Python')
depsextras = dist.requires(extras=[extra])
for dep in reversed(depsextras):
if dep in deps:
depsextras.remove(dep)
deps = depsextras
for dep in deps:
for spec in dep.specs:
if spec[0] == '!=':
print('Conflicts:\t{} {} {}'.format(dep.key, '==', spec[1]))
else:
print('Requires:\t{} {} {}'.format(dep.key, spec[0], spec[1]))
print('%%description\t{}'.format(extra))
print('{} extra for {} python package'.format(extra, dist.key))
print('%%files\t\textras-{}\n'.format(extra))
if args.conflicts:
# Should we really add conflicts for extras?
# Creating a meta package per extra with recommends on, which has
# the requires/conflicts in stead might be a better solution...
for dep in dist.requires(extras=dist.extras):
name = dep.key
for spec in dep.specs:
if spec[0] == '!=':
if name not in py_deps:
py_deps[name] = []
spec = ('==', spec[1])
if spec not in py_deps[name]:
py_deps[name].append(spec)
names = list(py_deps.keys())
names.sort()
for name in names:
if py_deps[name]:
# Print out versioned provides, requires, recommends, conflicts
spec_list = []
for spec in py_deps[name]:
spec_list.append(convert(name, spec[0], spec[1]))
if len(spec_list) == 1:
print(spec_list[0])
else:
# Sort spec_list so that the results can be tested easily
print('({})'.format(' with '.join(sorted(spec_list))))
else:
# Print out unversioned provides, requires, recommends, conflicts
print(name)