From 48c0de39d9adfd239da80164da62bbdc995e41b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 26 Jun 2020 12:57:41 +0200 Subject: [PATCH] Add a script to generate Python bundled provides See https://src.fedoraproject.org/rpms/python-setuptools/pull-request/40 Strictly speaking, this is not an RPM generator, but: - it generates provides - it is tighly coupled with pythondistdeps.py Usage: 1. Run `$ /usr/lib/rpm/pythonbundles.py .../vendored.txt` 2. Copy the output into the spec as a macro definition: %global bundled %{expand: Provides: bundled(python3dist(appdirs)) = 1.4.3 Provides: bundled(python3dist(packaging)) = 16.8 Provides: bundled(python3dist(pyparsing)) = 2.2.1 Provides: bundled(python3dist(six)) = 1.15 } 3. Use the macro to expand the provides 4. Verify the macro contents in %check: %check ... %{_rpmconfigdir}/pythonbundles.py src/_vendor/vendored.txt --compare-with '%{bundled}' --- python-rpm-generators.spec | 7 +- pythonbundles.py | 90 +++++++++++++++++ tests/data/scripts_pythonbundles/pip.in | 24 +++++ tests/data/scripts_pythonbundles/pip.out | 24 +++++ tests/data/scripts_pythonbundles/pipenv.in | 59 +++++++++++ tests/data/scripts_pythonbundles/pipenv.out | 58 +++++++++++ .../scripts_pythonbundles/pkg_resources.in | 4 + .../scripts_pythonbundles/pkg_resources.out | 4 + tests/test_scripts_pythonbundles.py | 99 +++++++++++++++++++ tests/tests.yml | 2 +- 10 files changed, 368 insertions(+), 3 deletions(-) create mode 100644 pythonbundles.py create mode 100644 tests/data/scripts_pythonbundles/pip.in create mode 100644 tests/data/scripts_pythonbundles/pip.out create mode 100644 tests/data/scripts_pythonbundles/pipenv.in create mode 100644 tests/data/scripts_pythonbundles/pipenv.out create mode 100644 tests/data/scripts_pythonbundles/pkg_resources.in create mode 100644 tests/data/scripts_pythonbundles/pkg_resources.out create mode 100644 tests/test_scripts_pythonbundles.py diff --git a/python-rpm-generators.spec b/python-rpm-generators.spec index 5e371ae..500aa80 100644 --- a/python-rpm-generators.spec +++ b/python-rpm-generators.spec @@ -12,6 +12,7 @@ Source1: python.attr Source2: pythondist.attr Source3: pythonname.attr Source4: pythondistdeps.py +Source5: pythonbundles.py BuildArch: noarch @@ -35,7 +36,7 @@ cp -a %{sources} . %install install -Dpm0644 -t %{buildroot}%{_fileattrsdir} *.attr -install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} pythondistdeps.py +install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} *.py %files -n python3-rpm-generators %license COPYING @@ -43,10 +44,12 @@ install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} pythondistdeps.py %{_fileattrsdir}/pythondist.attr %{_fileattrsdir}/pythonname.attr %{_rpmconfigdir}/pythondistdeps.py +%{_rpmconfigdir}/pythonbundles.py %changelog -* Wed Jun 17 2020 Miro Hrončok - 11-8 +* Fri Jun 26 2020 Miro Hrončok - 11-8 - Fix python(abi) requires generator, it picked files from almost good directories +- Add a script to generate Python bundled provides * Thu May 21 2020 Miro Hrončok - 11-7 - Use PEP 503 names for requires diff --git a/pythonbundles.py b/pythonbundles.py new file mode 100644 index 0000000..5093dff --- /dev/null +++ b/pythonbundles.py @@ -0,0 +1,90 @@ +#!/usr/bin/python3 -B +# (imports pythondistdeps from /usr/lib/rpm, hence -B) +# +# This program is free software. +# +# It is placed in the public domain or under the CC0-1.0-Universal license, +# whichever is more permissive. +# +# Alternatively, it may be redistributed and/or modified under the terms of +# the LGPL version 2.1 (or later) or GPL version 2 (or later). +# +# Use this script to generate bundled provides, e.g.: +# ./pythonbundles.py setuptools-47.1.1/pkg_resources/_vendor/vendored.txt + +import pathlib +import sys + +# inject parse_version import to pythondistdeps +# not the nicest API, but :/ +from pkg_resources import parse_version +import pythondistdeps +pythondistdeps.parse_version = parse_version + + +def generate_bundled_provides(path, namespace): + provides = set() + + for line in path.read_text().splitlines(): + line, _, comment = line.partition('#') + if comment.startswith('egg='): + # not a real comment + # e.g. git+https://github.com/monty/spam.git@master#egg=spam&... + egg, *_ = comment.strip().partition(' ') + egg, *_ = egg.strip().partition('&') + name = pythondistdeps.normalize_name(egg[4:]) + provides.add(f'Provides: bundled({namespace}({name}))') + continue + line = line.strip() + if line: + name, _, version = line.partition('==') + name = pythondistdeps.normalize_name(name) + bundled_name = f"bundled({namespace}({name}))" + python_provide = pythondistdeps.convert(bundled_name, '==', version) + provides.add(f'Provides: {python_provide}') + + return provides + + +def compare(expected, given): + stripped = (l.strip() for l in given) + no_comments = set(l for l in stripped if not l.startswith('#')) + no_comments.discard('') + if expected == no_comments: + return True + extra_expected = expected - no_comments + extra_given = no_comments - expected + if extra_expected: + print('Missing expected provides:', file=sys.stderr) + for provide in sorted(extra_expected): + print(f' - {provide}', file=sys.stderr) + if extra_given: + print('Redundant unexpected provides:', file=sys.stderr) + for provide in sorted(extra_given): + print(f' + {provide}', file=sys.stderr) + return False + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser(prog=sys.argv[0], + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('vendored', metavar='VENDORED.TXT', + help='Upstream information about vendored libraries') + parser.add_argument('-c', '--compare-with', action='store', + help='A string value to compare with and verify') + parser.add_argument('-n', '--namespace', action='store', + help='What namespace of provides will used', default='python3dist') + args = parser.parse_args() + + provides = generate_bundled_provides(pathlib.Path(args.vendored), args.namespace) + + if args.compare_with: + given = args.compare_with.splitlines() + same = compare(provides, given) + if not same: + sys.exit(1) + else: + for provide in sorted(provides): + print(provide) diff --git a/tests/data/scripts_pythonbundles/pip.in b/tests/data/scripts_pythonbundles/pip.in new file mode 100644 index 0000000..74ecca4 --- /dev/null +++ b/tests/data/scripts_pythonbundles/pip.in @@ -0,0 +1,24 @@ +appdirs==1.4.3 +CacheControl==0.12.6 +colorama==0.4.3 +contextlib2==0.6.0.post1 +distlib==0.3.0 +distro==1.5.0 +html5lib==1.0.1 +ipaddress==1.0.23 # Only needed on 2.6 and 2.7 +msgpack==1.0.0 +packaging==20.3 +pep517==0.8.2 +progress==1.5 +pyparsing==2.4.7 +requests==2.23.0 + certifi==2020.04.05.1 + chardet==3.0.4 + idna==2.9 + urllib3==1.25.8 +resolvelib==0.3.0 +retrying==1.3.3 +setuptools==44.0.0 +six==1.14.0 +toml==0.10.0 +webencodings==0.5.1 diff --git a/tests/data/scripts_pythonbundles/pip.out b/tests/data/scripts_pythonbundles/pip.out new file mode 100644 index 0000000..0be59de --- /dev/null +++ b/tests/data/scripts_pythonbundles/pip.out @@ -0,0 +1,24 @@ +Provides: bundled(python3dist(appdirs)) = 1.4.3 +Provides: bundled(python3dist(cachecontrol)) = 0.12.6 +Provides: bundled(python3dist(certifi)) = 2020.4.5.1 +Provides: bundled(python3dist(chardet)) = 3.0.4 +Provides: bundled(python3dist(colorama)) = 0.4.3 +Provides: bundled(python3dist(contextlib2)) = 0.6^post1 +Provides: bundled(python3dist(distlib)) = 0.3 +Provides: bundled(python3dist(distro)) = 1.5 +Provides: bundled(python3dist(html5lib)) = 1.0.1 +Provides: bundled(python3dist(idna)) = 2.9 +Provides: bundled(python3dist(ipaddress)) = 1.0.23 +Provides: bundled(python3dist(msgpack)) = 1 +Provides: bundled(python3dist(packaging)) = 20.3 +Provides: bundled(python3dist(pep517)) = 0.8.2 +Provides: bundled(python3dist(progress)) = 1.5 +Provides: bundled(python3dist(pyparsing)) = 2.4.7 +Provides: bundled(python3dist(requests)) = 2.23 +Provides: bundled(python3dist(resolvelib)) = 0.3 +Provides: bundled(python3dist(retrying)) = 1.3.3 +Provides: bundled(python3dist(setuptools)) = 44 +Provides: bundled(python3dist(six)) = 1.14 +Provides: bundled(python3dist(toml)) = 0.10 +Provides: bundled(python3dist(urllib3)) = 1.25.8 +Provides: bundled(python3dist(webencodings)) = 0.5.1 diff --git a/tests/data/scripts_pythonbundles/pipenv.in b/tests/data/scripts_pythonbundles/pipenv.in new file mode 100644 index 0000000..de5797a --- /dev/null +++ b/tests/data/scripts_pythonbundles/pipenv.in @@ -0,0 +1,59 @@ +appdirs==1.4.4 +backports.shutil_get_terminal_size==1.0.0 +backports.weakref==1.0.post1 +click==7.1.2 +click-completion==0.5.2 +click-didyoumean==0.0.3 +colorama==0.4.3 +delegator.py==0.1.1 + pexpect==4.8.0 + ptyprocess==0.6.0 +python-dotenv==0.10.3 +first==2.0.1 +iso8601==0.1.12 +jinja2==2.11.2 +markupsafe==1.1.1 +parse==1.15.0 +pathlib2==2.3.5 + scandir==1.10 +pipdeptree==0.13.2 +pipreqs==0.4.10 + docopt==0.6.2 + yarg==0.1.9 +pythonfinder==1.2.4 +requests==2.23.0 + chardet==3.0.4 + idna==2.9 + urllib3==1.25.9 + certifi==2020.4.5.1 +requirementslib==1.5.11 + attrs==19.3.0 + distlib==0.3.0 + packaging==20.3 + pyparsing==2.4.7 + plette==0.2.3 + tomlkit==0.5.11 +shellingham==1.3.2 +six==1.14.0 +semver==2.9.0 +toml==0.10.1 +cached-property==1.5.1 +vistir==0.5.2 +pip-shims==0.5.2 + contextlib2==0.6.0.post1 + funcsigs==1.0.2 +enum34==1.1.10 +# yaspin==0.15.0 +yaspin==0.14.3 +cerberus==1.3.2 +resolvelib==0.3.0 +backports.functools_lru_cache==1.6.1 +pep517==0.8.2 + zipp==0.6.0 + importlib_metadata==1.6.0 + importlib-resources==1.5.0 + more-itertools==5.0.0 +git+https://github.com/sarugaku/passa.git@master#egg=passa +orderedmultidict==1.0.1 +dparse==0.5.0 +python-dateutil==2.8.1 diff --git a/tests/data/scripts_pythonbundles/pipenv.out b/tests/data/scripts_pythonbundles/pipenv.out new file mode 100644 index 0000000..524ef72 --- /dev/null +++ b/tests/data/scripts_pythonbundles/pipenv.out @@ -0,0 +1,58 @@ +Provides: bundled(python3dist(appdirs)) = 1.4.4 +Provides: bundled(python3dist(attrs)) = 19.3 +Provides: bundled(python3dist(backports-functools-lru-cache)) = 1.6.1 +Provides: bundled(python3dist(backports-shutil-get-terminal-size)) = 1 +Provides: bundled(python3dist(backports-weakref)) = 1^post1 +Provides: bundled(python3dist(cached-property)) = 1.5.1 +Provides: bundled(python3dist(cerberus)) = 1.3.2 +Provides: bundled(python3dist(certifi)) = 2020.4.5.1 +Provides: bundled(python3dist(chardet)) = 3.0.4 +Provides: bundled(python3dist(click)) = 7.1.2 +Provides: bundled(python3dist(click-completion)) = 0.5.2 +Provides: bundled(python3dist(click-didyoumean)) = 0.0.3 +Provides: bundled(python3dist(colorama)) = 0.4.3 +Provides: bundled(python3dist(contextlib2)) = 0.6^post1 +Provides: bundled(python3dist(delegator-py)) = 0.1.1 +Provides: bundled(python3dist(distlib)) = 0.3 +Provides: bundled(python3dist(docopt)) = 0.6.2 +Provides: bundled(python3dist(dparse)) = 0.5 +Provides: bundled(python3dist(enum34)) = 1.1.10 +Provides: bundled(python3dist(first)) = 2.0.1 +Provides: bundled(python3dist(funcsigs)) = 1.0.2 +Provides: bundled(python3dist(idna)) = 2.9 +Provides: bundled(python3dist(importlib-metadata)) = 1.6 +Provides: bundled(python3dist(importlib-resources)) = 1.5 +Provides: bundled(python3dist(iso8601)) = 0.1.12 +Provides: bundled(python3dist(jinja2)) = 2.11.2 +Provides: bundled(python3dist(markupsafe)) = 1.1.1 +Provides: bundled(python3dist(more-itertools)) = 5 +Provides: bundled(python3dist(orderedmultidict)) = 1.0.1 +Provides: bundled(python3dist(packaging)) = 20.3 +Provides: bundled(python3dist(parse)) = 1.15 +Provides: bundled(python3dist(passa)) +Provides: bundled(python3dist(pathlib2)) = 2.3.5 +Provides: bundled(python3dist(pep517)) = 0.8.2 +Provides: bundled(python3dist(pexpect)) = 4.8 +Provides: bundled(python3dist(pip-shims)) = 0.5.2 +Provides: bundled(python3dist(pipdeptree)) = 0.13.2 +Provides: bundled(python3dist(pipreqs)) = 0.4.10 +Provides: bundled(python3dist(plette)) = 0.2.3 +Provides: bundled(python3dist(ptyprocess)) = 0.6 +Provides: bundled(python3dist(pyparsing)) = 2.4.7 +Provides: bundled(python3dist(python-dateutil)) = 2.8.1 +Provides: bundled(python3dist(python-dotenv)) = 0.10.3 +Provides: bundled(python3dist(pythonfinder)) = 1.2.4 +Provides: bundled(python3dist(requests)) = 2.23 +Provides: bundled(python3dist(requirementslib)) = 1.5.11 +Provides: bundled(python3dist(resolvelib)) = 0.3 +Provides: bundled(python3dist(scandir)) = 1.10 +Provides: bundled(python3dist(semver)) = 2.9 +Provides: bundled(python3dist(shellingham)) = 1.3.2 +Provides: bundled(python3dist(six)) = 1.14 +Provides: bundled(python3dist(toml)) = 0.10.1 +Provides: bundled(python3dist(tomlkit)) = 0.5.11 +Provides: bundled(python3dist(urllib3)) = 1.25.9 +Provides: bundled(python3dist(vistir)) = 0.5.2 +Provides: bundled(python3dist(yarg)) = 0.1.9 +Provides: bundled(python3dist(yaspin)) = 0.14.3 +Provides: bundled(python3dist(zipp)) = 0.6 diff --git a/tests/data/scripts_pythonbundles/pkg_resources.in b/tests/data/scripts_pythonbundles/pkg_resources.in new file mode 100644 index 0000000..7f4f408 --- /dev/null +++ b/tests/data/scripts_pythonbundles/pkg_resources.in @@ -0,0 +1,4 @@ +packaging==16.8 +pyparsing==2.2.1 +six==1.10.0 +appdirs==1.4.3 diff --git a/tests/data/scripts_pythonbundles/pkg_resources.out b/tests/data/scripts_pythonbundles/pkg_resources.out new file mode 100644 index 0000000..294ad86 --- /dev/null +++ b/tests/data/scripts_pythonbundles/pkg_resources.out @@ -0,0 +1,4 @@ +Provides: bundled(python3dist(appdirs)) = 1.4.3 +Provides: bundled(python3dist(packaging)) = 16.8 +Provides: bundled(python3dist(pyparsing)) = 2.2.1 +Provides: bundled(python3dist(six)) = 1.10 diff --git a/tests/test_scripts_pythonbundles.py b/tests/test_scripts_pythonbundles.py new file mode 100644 index 0000000..c04230e --- /dev/null +++ b/tests/test_scripts_pythonbundles.py @@ -0,0 +1,99 @@ +# Run tests using pytest, e.g. from the root directory +# $ python3 -m pytest --ignore tests/testing/ -vvv +# +# Requirements for this script: +# - Python >= 3.6 +# - pytest +import pathlib +import pytest +import random +import sys +import subprocess + +PYTHONBUNDLES = pathlib.Path(__file__).parent / '..' / 'pythonbundles.py' +TEST_DATA = pathlib.Path(__file__).parent / 'data' / 'scripts_pythonbundles' + + +def run_pythonbundles(*args, success=True): + """ + Runs pythonbundles.py with given command line arguments + + Arguments: + *args: Shell arguments passed to the script + success: + - true-ish: assert return code is 0 (default) + - false-ish (excluding None): assert return code is not 0 + - None: don't assert return code value + """ + cp = subprocess.run((sys.executable, PYTHONBUNDLES, *args), encoding='utf-8', + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if success: + assert cp.returncode == 0, cp.stderr + elif success is not None: + assert cp.returncode != 0, cp.stdout + return cp + + +projects = pytest.mark.parametrize('project', ('pkg_resources', 'pip', 'pipenv')) + + +@projects +def test_output_consistency(project): + cp = run_pythonbundles(TEST_DATA / f'{project}.in') + expected = (TEST_DATA / f'{project}.out').read_text() + assert cp.stdout == expected, cp.stdout + assert cp.stderr == '', cp.stderr + + +@pytest.mark.parametrize('namespace', ('python2dist', 'python3.11dist', 'pypy2.7dist')) +@projects +def test_namespace(project, namespace): + cp = run_pythonbundles(TEST_DATA / f'{project}.in', f'--namespace={namespace}') + expected = (TEST_DATA / f'{project}.out').read_text().replace('python3dist', namespace) + assert cp.stdout == expected, cp.stdout + assert cp.stderr == '', cp.stderr + + +@projects +def test_compare_with_identical(project): + expected = (TEST_DATA / f'{project}.out').read_text() + cp = run_pythonbundles(TEST_DATA / f'{project}.in', '--compare-with', expected) + assert cp.stdout == '', cp.stdout + assert cp.stderr == '', cp.stderr + + +@projects +def test_compare_with_shuffled(project): + expected = (TEST_DATA / f'{project}.out').read_text() + lines = expected.splitlines() + # some extra whitespace and comments + lines[0] = f' {lines[0]} ' + lines.extend([''] * 3) + lines.append('# this is a comment on a single line') + random.shuffle(lines) + shuffled = '\n'.join(lines) + cp = run_pythonbundles(TEST_DATA / f'{project}.in', '--compare-with', shuffled) + assert cp.stdout == '', cp.stdout + assert cp.stderr == '', cp.stderr + + +@projects +def test_compare_with_missing(project): + expected = (TEST_DATA / f'{project}.out').read_text() + lines = expected.splitlines() + missing = lines[0] + del lines[0] + shorter = '\n'.join(lines) + cp = run_pythonbundles(TEST_DATA / f'{project}.in', '--compare-with', shorter, success=False) + assert cp.stdout == '', cp.stdout + assert cp.stderr == f'Missing expected provides:\n - {missing}\n', cp.stderr + + +@projects +def test_compare_with_unexpected(project): + expected = (TEST_DATA / f'{project}.out').read_text() + unexpected = 'Provides: bundled(python3dist(brainfuck)) = 6.6.6' + longer = f'{expected}{unexpected}\n' + cp = run_pythonbundles(TEST_DATA / f'{project}.in', '--compare-with', longer, success=False) + assert cp.stdout == '', cp.stdout + assert cp.stderr == f'Redundant unexpected provides:\n + {unexpected}\n', cp.stderr diff --git a/tests/tests.yml b/tests/tests.yml index d07e24f..a79a5f1 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -30,7 +30,7 @@ - prepare-test-data: dir: . run: tar -xvf test-sources-*.tar.gz -C ./tests/data/scripts_pythondistdeps/ - - pythondistdeps: + - pytest: dir: ./tests # Use update-test-sources.sh to update the test data run: python3 -m pytest --capture=no -vvv