Support defining %py3_shebang_flags to %nil

Also add %python_platform_triplet, %python_ext_suffix and %python_platform.

The CI tests are limited to x86_64 for now.

https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names

compileall2.py

@@ -113,6 +113,13 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
     hardlink_dupes: hardlink duplicated pyc files
     """
     ProcessPoolExecutor = None
+    if ddir is not None and (stripdir is not None or prependdir is not None):
+        raise ValueError(("Destination dir (ddir) cannot be used "
+                          "in combination with stripdir or prependdir"))
+    if ddir is not None:
+        stripdir = dir
+        prependdir = ddir
+        ddir = None
     if workers is not None:
         if workers < 0:
             raise ValueError('workers must be greater or equal to 0')

macros.python

@@ -4,10 +4,21 @@
 %python_sitearch %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")
 %python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
 %python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
+%python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")
+%python_platform_triplet %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
+%python_ext_suffix %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
 
 %py_setup setup.py
 %py_shbang_opts -s
 %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-})
+%py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-})
+%py_shebang_fix %{expand:\\\
+  if [ -z "%{?py_shebang_flags}" ]; then
+    shebang_flags="-k"
+  else
+    shebang_flags="-ka%{py_shebang_flags}"
+  fi
+  /usr/bin/pathfix.py -pni %{__python} $shebang_flags}
 
 # Use the slashes after expand so that the command starts on the same line as
 # the macro

macros.python-srpm

@@ -93,7 +93,7 @@
 # Accepts zero to three arguments:
 # 1:  The PyPI project name, defaulting to %srcname if it is defined, then
 #     %pypi_name if it is defined, then just %name.
-# 2:  The PYPI version, defaulting to %version.
+# 2:  The PYPI version, defaulting to %version with tildes stripped.
 # 3:  The file extension, defaulting to "tar.gz".  (A period will be added
 #     automatically.)
 # Requires %__pypi_url and %__pypi_default_extension to be defined.
@@ -120,7 +120,7 @@
 \
     -- If no second argument, use %version
     if ver == '%2' then
-        ver = rpm.expand('%version')
+        ver = rpm.expand('%version'):gsub('~', '')
     end
 \
     -- If no third argument, use the preset default extension
@@ -132,3 +132,20 @@
 \
     print(url .. first .. '/' .. src .. '/' .. src .. '-' .. ver .. '.' .. ext)
 }
+
+%py_provides() %{lua:
+    local name = rpm.expand('%1')
+    if name == '%1' then
+        rpm.expand('%{error:%%py_provides requires at least 1 argument, the name to provide}')
+    end
+    local evr = rpm.expand('%2')
+    if evr == '%2' then
+        evr = rpm.expand('%{?epoch:%{epoch}:}%{version}-%{release}')
+    end
+    print('Provides: ' .. name .. ' = ' .. evr .. '\\n')
+    -- NB: dash needs to be escaped!
+    if name:match('^python3%-') then
+      replaced = name:gsub('^python3%-', 'python-')
+      print('Provides: ' .. replaced .. ' = ' .. evr .. '\\n')
+    end
+}

macros.python2

@@ -5,6 +5,14 @@
 
 %py2_shbang_opts -s
 %py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-})
+%py2_shebang_flags %(opts=%{py2_shbang_opts}; echo ${opts#-})
+%py2_shebang_fix %{expand:\\\
+  if [ -z "%{?py_shebang_flags}" ]; then
+    shebang_flags="-k"
+  else
+    shebang_flags="-ka%{py2_shebang_flags}"
+  fi
+  /usr/bin/pathfix.py -pni %{__python2} $shebang_flags}
 
 # Use the slashes after expand so that the command starts on the same line as
 # the macro

macros.python3

@@ -3,10 +3,20 @@
 %python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
 %python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
 %python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")
+%python3_platform_triplet %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
+%python3_ext_suffix %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
 %py3dir %{_builddir}/python3-%{name}-%{version}-%{release}
 
 %py3_shbang_opts -s
 %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-})
+%py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-})
+%py3_shebang_fix %{expand:\\\
+  if [ -z "%{?py3_shebang_flags}" ]; then
+    shebang_flags="-k"
+  else
+    shebang_flags="-ka%{py3_shebang_flags}"
+  fi
+  /usr/bin/pathfix.py -pni %{__python3} $shebang_flags}
 
 # Use the slashes after expand so that the command starts on the same line as
 # the macro
@@ -45,7 +55,7 @@
 # This only supports Python 3.5+ and will never work with Python 2.
 # Hence, it has no Python version in the name.
 %pycached() %{lua:
-  path = rpm.expand("%{?1}")
+  path = rpm.expand("%{?*}")
   if (string.sub(path, "-3") ~= ".py") then
     rpm.expand("%{error:%%pycached can only be used with paths explicitly ending with .py}")
   else
@@ -56,3 +66,12 @@
     print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc")
   end
 }
+
+# This is intended for Python 3 only, hence also no Python version in the name.
+%__pytest /usr/bin/pytest
+%pytest %{expand:\\\
+  CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\
+  PATH="%{buildroot}%{_bindir}:$PATH"\\\
+  PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\
+  PYTHONDONTWRITEBYTECODE=1\\\
+  %__pytest}

python-rpm-macros.spec

@@ -1,6 +1,6 @@
 Name:           python-rpm-macros
 Version:        3
-Release:        54%{?dist}
+Release:        60%{?dist}
 Summary:        The unversioned Python RPM macros
 
 # macros: MIT, compileall2.py: PSFv2
@@ -10,7 +10,7 @@ Source1:        macros.python-srpm
 Source2:        macros.python2
 Source3:        macros.python3
 Source4:        macros.pybytecompile
-Source5:        https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py
+Source5:        https://github.com/fedora-python/compileall2/raw/v0.7.1/compileall2.py
 
 BuildArch:      noarch
 # For %%python3_pkgversion used in %%python_provide and compileall2.py
@@ -35,6 +35,7 @@ RPM macros for building Python source packages.
 %package -n python2-rpm-macros
 Summary:        RPM macros for building Python 2 packages
 Requires:       python-srpm-macros >= 3-38
+Requires:       python-rpm-macros
 # Would need to be different for each release - worth it?
 #Conflicts:      python2-devel < 2.7.11-3
 
@@ -44,6 +45,7 @@ RPM macros for building Python 2 packages.
 %package -n python3-rpm-macros
 Summary:        RPM macros for building Python 3 packages
 Requires:       python-srpm-macros >= 3-38
+Requires:       python-rpm-macros
 
 %description -n python3-rpm-macros
 RPM macros for building Python 3 packages.
@@ -78,6 +80,28 @@ install -m 644 %{SOURCE5} \
 
 
 %changelog
+* Tue Dec 08 2020 Miro Hrončok <mhroncok@redhat.com> - 3-60
+- Support defining %%py3_shebang_flags to %%nil
+
+* Thu Sep 24 2020 Miro Hrončok <mhroncok@redhat.com> - 3-59
+- Add %%python3_platform_triplet and %%python3_ext_suffix
+- https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
+
+* Mon Jun 15 2020 Miro Hrončok <mhroncok@redhat.com> - 3-58
+- Allow to combine %%pycached with other macros (e.g. %%exclude or %%ghost) (#1838992)
+
+* Wed May 20 2020 Miro Hrončok <mhroncok@redhat.com> - 3-57
+- Implement %%py_provides
+- Implement %%pytest
+- Implement %%pyX_shebang_fix
+- Strip tildes from %%version in %%pypi_source by default
+
+* Tue Apr 28 2020 Miro Hrončok <mhroncok@redhat.com> - 3-56
+- Make pythonX-rpm-macros depend on python-rpm-macros (#1827811)
+
+* Tue Mar 03 2020 Lumír Balhar <lbalhar@redhat.com> - 3-55
+- Update of bundled compileall2 module to 0.7.1 (bugfix release)
+
 * Mon Feb 10 2020 Miro Hrončok <mhroncok@redhat.com> - 3-54
 - Update of bundled compileall2 module to 0.7.0
   Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication

tests/.gitignore

@@ -0,0 +1 @@
+__*__/

tests/test_evals.py

@@ -0,0 +1,256 @@
+import os
+import subprocess
+import platform
+import sys
+
+import pytest
+
+X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}'
+XY = f'{sys.version_info[0]}{sys.version_info[1]}'
+
+
+def rpm_eval(expression, fails=False, **kwargs):
+    cmd = ['rpmbuild']
+    for var, value in kwargs.items():
+        if value is None:
+            cmd += ['--undefine', var]
+        else:
+            cmd += ['--define', f'{var} {value}']
+    cmd += ['--eval', expression]
+    cp = subprocess.run(cmd, text=True, env={**os.environ, 'LANG': 'C.utf-8'},
+                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    if fails:
+        assert cp.returncode != 0, cp.stdout
+    elif fails is not None:
+        assert cp.returncode == 0, cp.stdout
+    return cp.stdout.strip().splitlines()
+
+
+def shell_stdout(script):
+    return subprocess.check_output(script,
+                                   env={**os.environ, 'LANG': 'C.utf-8'},
+                                   text=True,
+                                   shell=True).rstrip()
+
+
+def test_python_provide_python():
+    assert rpm_eval('%python_provide python-foo') == []
+
+
+def test_python_provide_python3():
+    lines = rpm_eval('%python_provide python3-foo', version='6', release='1.fc66')
+    assert 'Obsoletes: python-foo < 6-1.fc66' in lines
+    assert 'Provides: python-foo = 6-1.fc66' in lines
+    assert len(lines) == 2
+
+
+def test_python_provide_python3_epoched():
+    lines = rpm_eval('%python_provide python3-foo', epoch='1', version='6', release='1.fc66')
+    assert 'Obsoletes: python-foo < 1:6-1.fc66' in lines
+    assert 'Provides: python-foo = 1:6-1.fc66' in lines
+    assert len(lines) == 2
+
+
+def test_python_provide_doubleuse():
+    lines = rpm_eval('%{python_provide python3-foo}%{python_provide python3-foo}',
+                     version='6', release='1.fc66')
+    assert 'Obsoletes: python-foo < 6-1.fc66' in lines
+    assert 'Provides: python-foo = 6-1.fc66' in lines
+    assert len(lines) == 4
+    assert len(set(lines)) == 2
+
+
+def test_py_provides_python():
+    lines = rpm_eval('%py_provides python-foo', version='6', release='1.fc66')
+    assert 'Provides: python-foo = 6-1.fc66' in lines
+    assert len(lines) == 1
+
+
+def test_py_provides_whatever():
+    lines = rpm_eval('%py_provides whatever', version='6', release='1.fc66')
+    assert 'Provides: whatever = 6-1.fc66' in lines
+    assert len(lines) == 1
+
+
+def test_py_provides_python3():
+    lines = rpm_eval('%py_provides python3-foo', version='6', release='1.fc66')
+    assert 'Provides: python3-foo = 6-1.fc66' in lines
+    assert 'Provides: python-foo = 6-1.fc66' in lines
+    assert len(lines) == 2
+
+
+def test_py_provides_python3_epoched():
+    lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66')
+    assert 'Provides: python3-foo = 1:6-1.fc66' in lines
+    assert 'Provides: python-foo = 1:6-1.fc66' in lines
+    assert len(lines) == 2
+
+
+def test_py_provides_doubleuse():
+    lines = rpm_eval('%{py_provides python3-foo}%{py_provides python3-foo}',
+                     version='6', release='1.fc66')
+    assert 'Provides: python3-foo = 6-1.fc66' in lines
+    assert 'Provides: python-foo = 6-1.fc66' in lines
+    assert len(lines) == 4
+    assert len(set(lines)) == 2
+
+
+def test_py_provides_with_evr():
+    lines = rpm_eval('%py_provides python3-foo 123',
+                     version='6', release='1.fc66')
+    assert 'Provides: python3-foo = 123' in lines
+    assert 'Provides: python-foo = 123' in lines
+    assert len(lines) == 2
+
+
+def test_pytest_passes_options_naturally():
+    lines = rpm_eval('%pytest -k foo')
+    assert '/usr/bin/pytest -k foo' in lines[-1]
+
+
+def test_pytest_different_command():
+    lines = rpm_eval('%pytest', __pytest='pytest-3')
+    assert 'pytest-3' in lines[-1]
+
+
+def test_pypi_source_default_name():
+    url = rpm_eval('%pypi_source',
+                   name='foo', version='6')[0]
+    assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'
+
+
+def test_pypi_source_default_srcname():
+    url = rpm_eval('%pypi_source',
+                   name='python-foo', srcname='foo', version='6')[0]
+    assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'
+
+
+def test_pypi_source_default_pypi_name():
+    url = rpm_eval('%pypi_source',
+                   name='python-foo', pypi_name='foo', version='6')[0]
+    assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'
+
+
+def test_pypi_source_default_name_uppercase():
+    url = rpm_eval('%pypi_source',
+                   name='Foo', version='6')[0]
+    assert url == 'https://files.pythonhosted.org/packages/source/F/Foo/Foo-6.tar.gz'
+
+
+def test_pypi_source_provided_name():
+    url = rpm_eval('%pypi_source foo',
+                   name='python-bar', pypi_name='bar', version='6')[0]
+    assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'
+
+
+def test_pypi_source_provided_name_version():
+    url = rpm_eval('%pypi_source foo 6',
+                   name='python-bar', pypi_name='bar', version='3')[0]
+    assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'
+
+
+def test_pypi_source_provided_name_version_ext():
+    url = rpm_eval('%pypi_source foo 6 zip',
+                   name='python-bar', pypi_name='bar', version='3')[0]
+    assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.zip'
+
+
+def test_pypi_source_prerelease():
+    url = rpm_eval('%pypi_source',
+                   name='python-foo', pypi_name='foo', version='6~b2')[0]
+    assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6b2.tar.gz'
+
+
+def test_pypi_source_explicit_tilde():
+    url = rpm_eval('%pypi_source foo 6~6',
+                   name='python-foo', pypi_name='foo', version='6')[0]
+    assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6~6.tar.gz'
+
+
+def test_py3_shebang_fix():
+    cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[-1].strip()
+    assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3'
+
+
+def test_py3_shebang_fix_default_shebang_flags():
+    lines = rpm_eval('%py3_shebang_fix arg1 arg2')
+    lines[-1] = 'echo $shebang_flags'
+    assert shell_stdout('\n'.join(lines)) == '-kas'
+
+
+def test_py3_shebang_fix_custom_shebang_flags():
+    lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags='Es')
+    lines[-1] = 'echo $shebang_flags'
+    assert shell_stdout('\n'.join(lines)) == '-kaEs'
+
+
+@pytest.mark.parametrize('flags', [None, '%{nil}'])
+def test_py3_shebang_fix_no_shebang_flags(flags):
+    lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags=flags)
+    lines[-1] = 'echo $shebang_flags'
+    assert shell_stdout('\n'.join(lines)) == '-k'
+
+
+def test_py_shebang_fix_custom_python():
+    cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[-1].strip()
+    assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3'
+
+
+def test_pycached_in_sitelib():
+    lines = rpm_eval('%pycached %{python3_sitelib}/foo*.py')
+    assert lines == [
+        f'/usr/lib/python{X_Y}/site-packages/foo*.py',
+        f'/usr/lib/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc'
+    ]
+
+
+def test_pycached_in_sitearch():
+    lines = rpm_eval('%pycached %{python3_sitearch}/foo*.py')
+    lib = rpm_eval('%_lib')[0]
+    assert lines == [
+        f'/usr/{lib}/python{X_Y}/site-packages/foo*.py',
+        f'/usr/{lib}/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc'
+    ]
+
+
+def test_pycached_in_36():
+    lines = rpm_eval('%pycached /usr/lib/python3.6/site-packages/foo*.py')
+    assert lines == [
+        '/usr/lib/python3.6/site-packages/foo*.py',
+        '/usr/lib/python3.6/site-packages/__pycache__/foo*.cpython-36{,.opt-?}.pyc'
+    ]
+
+
+def test_pycached_in_custom_dir():
+    lines = rpm_eval('%pycached /bar/foo*.py')
+    assert lines == [
+        '/bar/foo*.py',
+        '/bar/__pycache__/foo*.cpython-3*{,.opt-?}.pyc'
+    ]
+
+
+def test_pycached_with_exclude():
+    lines = rpm_eval('%pycached %exclude %{python3_sitelib}/foo*.py')
+    assert lines == [
+        f'%exclude /usr/lib/python{X_Y}/site-packages/foo*.py',
+        f'%exclude /usr/lib/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc'
+    ]
+
+
+def test_pycached_fails_with_extension_glob():
+    lines = rpm_eval('%pycached %{python3_sitelib}/foo.py*', fails=True)
+    assert lines[0] == 'error: %pycached can only be used with paths explicitly ending with .py'
+
+
+# we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64
+x86_64_only = pytest.mark.skipif(platform.machine() != "x86_64", reason="works on x86_64 only")
+
+
+@x86_64_only
+def test_platform_triplet():
+    assert rpm_eval("%python3_platform_triplet")[0] == "x86_64-linux-gnu"
+
+
+@x86_64_only
+def test_ext_suffix():
+    assert rpm_eval("%python3_ext_suffix")[0] == f".cpython-{XY}-x86_64-linux-gnu.so"

tests/tests.yml

@@ -0,0 +1,23 @@
+---
+- hosts: localhost
+  tags:
+    - classic
+  tasks:
+    - dnf:
+        name: "*"
+        state: latest
+
+- hosts: localhost
+  roles:
+  - role: standard-test-basic
+    tags:
+    - classic
+    tests:
+    - pytest:
+        dir: .
+        run: pytest -v
+    required_packages:
+    - rpm-build
+    - python-rpm-macros
+    - python3-rpm-macros
+    - python3-pytest