Disable certain rpminspect inspections not relevant to this package

This is in-line with https://fedoraproject.org/wiki/Changes/PythonSafePath#Opting_out

Backported from https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/141

import_all_modules.py

@@ -0,0 +1,171 @@
+'''Script to perform import of each module given to %%py_check_import
+'''
+import argparse
+import importlib
+import fnmatch
+import os
+import re
+import site
+import sys
+
+from contextlib import contextmanager
+from pathlib import Path
+
+
+def read_modules_files(file_paths):
+    '''Read module names from the files (modules must be newline separated).
+
+    Return the module names list or, if no files were provided, an empty list.
+    '''
+
+    if not file_paths:
+        return []
+
+    modules = []
+    for file in file_paths:
+        file_contents = file.read_text()
+        modules.extend(file_contents.split())
+    return modules
+
+
+def read_modules_from_cli(argv):
+    '''Read module names from command-line arguments (space or comma separated).
+
+    Return the module names list.
+    '''
+
+    if not argv:
+        return []
+
+    # %%py3_check_import allows to separate module list with comma or whitespace,
+    # we need to unify the output to a list of particular elements
+    modules_as_str = ' '.join(argv)
+    modules = re.split(r'[\s,]+', modules_as_str)
+    # Because of shell expansion in some less typical cases it may happen
+    # that a trailing space will occur at the end of the list.
+    # Remove the empty items from the list before passing it further
+    modules = [m for m in modules if m]
+    return modules
+
+
+def filter_top_level_modules_only(modules):
+    '''Filter out entries with nested modules (containing dot) ie. 'foo.bar'.
+
+    Return the list of top-level modules.
+    '''
+
+    return [module for module in modules if '.' not in module]
+
+
+def any_match(text, globs):
+    '''Return True if any of given globs fnmatchcase's the given text.'''
+
+    return any(fnmatch.fnmatchcase(text, g) for g in globs)
+
+
+def exclude_unwanted_module_globs(globs, modules):
+    '''Filter out entries which match the either of the globs given as argv.
+
+    Return the list of filtered modules.
+    '''
+
+    return [m for m in modules if not any_match(m, globs)]
+
+
+def read_modules_from_all_args(args):
+    '''Return a joined list of modules from all given command-line arguments.
+    '''
+
+    modules = read_modules_files(args.filename)
+    modules.extend(read_modules_from_cli(args.modules))
+    if args.exclude:
+        modules = exclude_unwanted_module_globs(args.exclude, modules)
+
+    if args.top_level:
+        modules = filter_top_level_modules_only(modules)
+
+    # Error when someone accidentally managed to filter out everything
+    if len(modules) == 0:
+        raise ValueError('No modules to check were left')
+
+    return modules
+
+
+def import_modules(modules):
+    '''Procedure to perform import check for each module name from the given list of modules.
+    '''
+
+    for module in modules:
+        print('Check import:', module, file=sys.stderr)
+        importlib.import_module(module)
+
+
+def argparser():
+    parser = argparse.ArgumentParser(
+        description='Generate list of all importable modules for import check.'
+    )
+    parser.add_argument(
+        'modules', nargs='*',
+        help=('Add modules to check the import (space or comma separated).'),
+    )
+    parser.add_argument(
+        '-f', '--filename', action='append', type=Path,
+        help='Add importable module names list from file.',
+    )
+    parser.add_argument(
+        '-t', '--top-level', action='store_true',
+        help='Check only top-level modules.',
+    )
+    parser.add_argument(
+        '-e', '--exclude', action='append',
+        help='Provide modules globs to be excluded from the check.',
+    )
+    return parser
+
+
+@contextmanager
+def remove_unwanteds_from_sys_path():
+    '''Remove cwd and this script's parent from sys.path for the import test.
+    Bring the original contents back after import is done (or failed)
+    '''
+
+    cwd_absolute = Path.cwd().absolute()
+    this_file_parent = Path(__file__).parent.absolute()
+    old_sys_path = list(sys.path)
+    for path in old_sys_path:
+        if Path(path).absolute() in (cwd_absolute, this_file_parent):
+            sys.path.remove(path)
+    try:
+        yield
+    finally:
+        sys.path = old_sys_path
+
+
+def addsitedirs_from_environ():
+    '''Load directories from the _PYTHONSITE environment variable (separated by :)
+    and load the ones already present in sys.path via site.addsitedir()
+    to handle .pth files in them.
+
+    This is needed to properly import old-style namespace packages with nspkg.pth files.
+    See https://bugzilla.redhat.com/2018551 for a more detailed rationale.'''
+    for path in os.getenv('_PYTHONSITE', '').split(':'):
+        if path in sys.path:
+            site.addsitedir(path)
+
+
+def main(argv=None):
+
+    cli_args = argparser().parse_args(argv)
+
+    if not cli_args.modules and not cli_args.filename:
+        raise ValueError('No modules to check were provided')
+
+    modules = read_modules_from_all_args(cli_args)
+
+    with remove_unwanteds_from_sys_path():
+        addsitedirs_from_environ()
+        import_modules(modules)
+
+
+if __name__ == '__main__':
+    main()

macros.python

@@ -1,15 +1,19 @@
 # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15
 # __python is defined to error by default in the srpm macros
-%python_sitelib %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))")
-%python_sitearch %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))")
-%python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
-%python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
-%python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")
-%python_platform_triplet %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
-%python_ext_suffix %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time)
+#     so we set it manually (to empty string), making our Python prefer the correct install scheme location
+%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))")
+%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))")
+%python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
+%python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
+%python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")
+%python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
+%python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+%python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)")
 
 %py_setup setup.py
-%py_shbang_opts -s
+%_py_shebang_s s
+%py_shbang_opts -%{?_py_shebang_s}
 %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-})
 %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-})
 %py_shebang_fix %{expand:\\\
@@ -67,16 +71,28 @@
 }
 
 # With $PATH and $PYTHONPATH set to the %%buildroot,
-# try to import the given Python module(s).
+# try to import the Python module(s) given as command-line args or read from file (-f).
+# Respect the custom values of %%py_shebang_flags or set nothing if it's undefined.
+# Filter and check import on only top-level modules using -t flag.
+# Exclude unwanted modules by passing their globs to -e option.
 # Useful as a smoke test in %%check when running tests is not feasible.
-# Use spaces or commas as separators.
-%py_check_import() %{expand:\\\
-  (cd %{_topdir} &&\\\
+# Use spaces or commas as separators if providing list directly.
+# Use newlines as separators if providing list in a file.
+%py_check_import(e:tf:) %{expand:\\\
   PATH="%{buildroot}%{_bindir}:$PATH"\\\
   PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\
+  _PYTHONSITE="%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}"\\\
   PYTHONDONTWRITEBYTECODE=1\\\
-  %{__python} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}"
-  )
+  %{lua:
+  local command = "%{__python} "
+  if rpm.expand("%{?py_shebang_flags}") ~= "" then
+    command = command .. "-%{py_shebang_flags}"
+  end
+  command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py "
+  -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809
+  local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ")
+  print(command .. args)
+  }
 }
 
 %python_provide() %{lua:

macros.python-srpm

@@ -49,6 +49,20 @@
 # Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks.
 %python3_pkgversion 3
 
+# Define where Python wheels will be stored and the prefix of -wheel packages
+# - In Fedora we want wheel subpackages named e.g. `python-pip-wheel` that
+#   install packages into `/usr/share/python-wheels`. Both names are not
+#   versioned, because they're used by all Python 3 stacks.
+# - In RHEL we want wheel packages named e.g. `python3-pip-wheel` and
+#   `python3.11-pip-wheel` that install packages into similarly versioned
+#   locations. We want each Python stack in RHEL to have their own wheels,
+#   because the main python3 wheels (which we can't upgrade) will likely be
+#   quite old by the time we're adding new alternate Python stacks.
+# - In ELN we want to follow Fedora, because builds for ELN and Fedora rawhide
+#   need to be interoperable.
+%python_wheel_pkg_prefix python%{?rhel:%{!?eln:%{python3_pkgversion}}}
+%python_wheel_dir %{_datadir}/%{python_wheel_pkg_prefix}-wheels
+
 
 ### BRP scripts (and related macros)
 

macros.python3

@@ -1,13 +1,17 @@
-%python3_sitelib %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))")
-%python3_sitearch %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))")
-%python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
-%python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
-%python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")
-%python3_platform_triplet %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
-%python3_ext_suffix %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time)
+#     so we set it manually (to empty string), making our Python prefer the correct install scheme location
+%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))")
+%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))")
+%python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
+%python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
+%python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")
+%python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
+%python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+%python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)")
 %py3dir %{_builddir}/python3-%{name}-%{version}-%{release}
 
-%py3_shbang_opts -s
+%_py3_shebang_s s
+%py3_shbang_opts -%{?_py3_shebang_s}
 %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-})
 %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-})
 %py3_shebang_fix %{expand:\\\
@@ -65,16 +69,28 @@
 }
 
 # With $PATH and $PYTHONPATH set to the %%buildroot,
-# try to import the given Python 3 module(s).
+# try to import the Python 3 module(s) given as command-line args or read from file (-f).
+# Respect the custom values of %%py3_shebang_flags or set nothing if it's undefined.
+# Filter and check import on only top-level modules using -t flag.
+# Exclude unwanted modules by passing their globs to -e option.
 # Useful as a smoke test in %%check when running tests is not feasible.
-# Use spaces or commas as separators.
-%py3_check_import() %{expand:\\\
-  (cd %{_topdir} &&\\\
+# Use spaces or commas as separators if providing list directly.
+# Use newlines as separators if providing list in a file.
+%py3_check_import(e:tf:) %{expand:\\\
   PATH="%{buildroot}%{_bindir}:$PATH"\\\
   PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\
+  _PYTHONSITE="%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}"\\\
   PYTHONDONTWRITEBYTECODE=1\\\
-  %{__python3} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}"
-  )
+  %{lua:
+  local command = "%{__python3} "
+  if rpm.expand("%{?py3_shebang_flags}") ~= "" then
+    command = command .. "-%{py3_shebang_flags}"
+  end
+  command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py "
+  -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809
+  local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ")
+  print(command .. args)
+  }
 }
 
 # This only supports Python 3.5+ and will never work with Python 2.
@@ -88,6 +104,7 @@
     pyminor = path:match("/python3.(%d+)/") or "*"
     dirname = path:match("(.*/)")
     modulename = path:match(".*/([^/]+).py")
+    -- %%python3_cache_tag is not used here because this macro supports not-installed CPythons
     print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc")
   end
 }

python-rpm-macros.spec

@@ -15,6 +15,7 @@ Source201:      python.lua
 # Python code
 %global compileall2_version 0.7.1
 Source301:      https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py
+Source302:      import_all_modules.py
 
 # BRP scripts
 # This one is from redhat-rpm-config < 190
@@ -31,6 +32,7 @@ Source402:      brp-python-hardlink
 Source403:      brp-fix-pyc-reproducibility
 
 # macros and lua: MIT
+# import_all_modules.py: MIT
 # compileall2.py: PSFv2
 # brp scripts: GPLv2+
 License:        MIT and Python and GPLv2+
@@ -47,7 +49,8 @@ elseif posix.stat('macros.python-srpm') then
 end
 }
 Version:        %{__default_python3_version}
-Release:        7%{?dist}
+Release:        12%{?dist}
+
 
 BuildArch:      noarch
 
@@ -84,7 +87,7 @@ Summary:        RPM macros for building Python 3 packages
 # For %%__python3 and %%python3
 Requires:       python-srpm-macros = %{version}-%{release}
 
-# For %%py_setup
+# For %%py_setup and import_all_modules.py
 Requires:       python-rpm-macros = %{version}-%{release}
 
 %description -n python3-rpm-macros
@@ -105,6 +108,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua
 
 mkdir -p %{buildroot}%{_rpmconfigdir}/redhat
 install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/
+install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/
 
 install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/
 
@@ -127,6 +131,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/
 %files
 %{rpmmacrodir}/macros.python
 %{rpmmacrodir}/macros.pybytecompile
+%{_rpmconfigdir}/redhat/import_all_modules.py
 
 %files -n python-srpm-macros
 %{rpmmacrodir}/macros.python-srpm
@@ -141,7 +146,32 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/
 
 
 %changelog
-* Fri Jul 23 2021 Fedora Release Engineering <releng@fedoraproject.org> - 3.9-7
+* Tue Jul 19 2022 Miro Hrončok <mhroncok@redhat.com> - 3.10-12
+- Define %%python3_cache_tag / %%python_cache_tag, e.g. cpython-311
+- Define and use %%{_py3_shebang_s} in the shebang macros for easier opt-out
+
+* Mon Feb 07 2022 Miro Hrončok <mhroncok@redhat.com> - 3.10-11
+- Set $RPM_BUILD_ROOT in %%{python3_...} macros
+  to allow selecting alternate sysconfig install scheme based on that variable
+
+* Mon Nov 01 2021 Karolina Surma <ksurma@redhat.com> - 3.10-10
+- Fix multiline arguments processing for %%py_check_import
+Resolves: rhbz#2018809
+- Fix %%py_shebang_flags handling within %%py_check_import
+Resolves: rhbz#2018615
+- Process .pth files in buildroot's sitedirs in %%py_check_import
+Resolves: rhbz#2018551
+- Move import_all_modules.py from python-srpm-macros to python-rpm-macros
+
+* Mon Oct 25 2021 Karolina Surma <ksurma@redhat.com> - 3.10-9
+- Introduce -f (read from file) option to %%py{3}_check_import
+- Introduce -t (filter top-level modules) option to %%py{3}_check_import
+- Introduce -e (exclude module globs) option to %%py{3}_check_import
+
+* Wed Sep 29 2021 Tomas Orsava <torsava@redhat.com> - 3.10-8
+- Define a new macros %%python_wheel_dir and %%python_wheel_pkg_prefix
+
+* Fri Jul 23 2021 Fedora Release Engineering <releng@fedoraproject.org> - 3.10-7
 - Rebuilt for https://fedoraproject.org/wiki/Fedora_35_Mass_Rebuild
 
 * Wed Jul 07 2021 Miro Hrončok <mhroncok@redhat.com> - 3.10-6

rpminspect.yaml

@@ -0,0 +1,7 @@
+# completely disabled inspections:
+inspections:
+    # there is no upstream and the files are changed from time to time
+    addedfiles: off
+    changedfiles: off
+    filesize: off
+    upstream: off

tests/test_evals.py

@@ -195,6 +195,24 @@ def test_py_provides_with_evr():
     assert len(lines) == 3
 
 
+def test_python_wheel_pkg_prefix():
+    assert rpm_eval('%python_wheel_pkg_prefix', fedora='44', rhel=None, eln=None) == ['python']
+    assert rpm_eval('%python_wheel_pkg_prefix', fedora='44', rhel=None, eln=None, python3_pkgversion='3.9') == ['python']
+    assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln='1') == ['python']
+    assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None) == ['python3']
+    assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None, python3_pkgversion='3.10') == ['python3.10']
+    assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None, python3_pkgversion='3.11') == ['python3.11']
+
+
+def test_python_wheel_dir():
+    assert rpm_eval('%python_wheel_dir', fedora='44', rhel=None, eln=None) == ['/usr/share/python-wheels']
+    assert rpm_eval('%python_wheel_dir', fedora='44', rhel=None, eln=None, python3_pkgversion='3.9') == ['/usr/share/python-wheels']
+    assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln='1') == ['/usr/share/python-wheels']
+    assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None) == ['/usr/share/python3-wheels']
+    assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None, python3_pkgversion='3.10') == ['/usr/share/python3.10-wheels']
+    assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None, python3_pkgversion='3.11') == ['/usr/share/python3.11-wheels']
+
+
 def test_pytest_passes_options_naturally():
     lines = rpm_eval('%pytest -k foo')
     assert '/usr/bin/pytest -k foo' in lines[-1]
@@ -316,6 +334,13 @@ def test_py3_shebang_fix_custom_shebang_flags():
     assert shell_stdout('\n'.join(lines)) == '-kaEs'
 
 
+@pytest.mark.parametrize('_py3_shebang_s', [None, '%{nil}'])
+def test_py3_shebang_fix_undefined_py3_shebang_s(_py3_shebang_s):
+    lines = rpm_eval('%py3_shebang_fix arg1 arg2', _py3_shebang_s=_py3_shebang_s)
+    lines[-1] = 'echo $shebang_flags'
+    assert shell_stdout('\n'.join(lines)) == '-k'
+
+
 @pytest.mark.parametrize('flags', [None, '%{nil}'])
 def test_py3_shebang_fix_no_shebang_flags(flags):
     lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags=flags)
@@ -542,6 +567,7 @@ unversioned_macros = pytest.mark.parametrize('macro', [
     '%python_platform',
     '%python_platform_triplet',
     '%python_ext_suffix',
+    '%python_cache_tag',
     '%py_shebang_fix',
     '%py_build',
     '%py_build_egg',
@@ -580,6 +606,18 @@ def test_ext_suffix():
     assert rpm_eval("%python3_ext_suffix") == [f".cpython-{XY}-x86_64-linux-gnu.so"]
 
 
+def test_cache_tag():
+    assert rpm_eval("%python3_cache_tag") == [f"cpython-{XY}"]
+
+
+def test_cache_tag_alternate_python():
+    assert rpm_eval("%python_cache_tag", __python=f"/usr/bin/python3.6") == [f"cpython-36"]
+
+
+def test_cache_tag_alternate_python3():
+    assert rpm_eval("%python3_cache_tag", __python3=f"/usr/bin/python3.6") == [f"cpython-36"]
+
+
 def test_python_sitelib_value():
     macro = '%python_sitelib'
     assert rpm_eval(macro, __python='/usr/bin/python3.6') == [f'/usr/lib/python3.6/site-packages']
@@ -605,23 +643,30 @@ def test_python3_sitearch_value(lib):
 
 
 @pytest.mark.parametrize(
-    'args, imports',
+    'args, expected_args',
     [
         ('six', 'six'),
-        ('five  six seven', 'five, six, seven'),
-        ('six,seven, eight', 'six, seven, eight'),
-        ('six.quarter  six.half,, SIX', 'six.quarter, six.half, SIX'),
+        ('-f foo.txt', '-f foo.txt'),
+        ('-t -f foo.txt six, seven', '-t -f foo.txt six, seven'),
+        ('-e "foo*" -f foo.txt six, seven', '-e "foo*" -f foo.txt six, seven'),
+        ('six.quarter six.half,, SIX', 'six.quarter six.half,, SIX'),
+        ('-f foo.txt six\nsix.half\nSIX', '-f foo.txt six six.half SIX'),
+        ('six \\ -e six.half', 'six -e six.half'),
     ]
 )
-@pytest.mark.parametrize('__python3', [None, f'/usr/bin/python{X_Y}', '/usr/bin/python3.6'])
-def test_py3_check_import(args, imports, __python3, lib):
+@pytest.mark.parametrize('__python3',
+                         [None,
+                          f'/usr/bin/python{X_Y}',
+                          '/usr/bin/python3.6'])
+def test_py3_check_import(args, expected_args, __python3, lib):
     x_y = X_Y
-    macors = {
+    macros = {
         'buildroot': 'BUILDROOT',
-        '_topdir': 'TOPDIR',
+        '_rpmconfigdir': 'RPMCONFIGDIR',
+        'py3_shebang_flags': 's',
     }
     if __python3 is not None:
-        macors['__python3'] = __python3
+        macros['__python3'] = __python3
         # If the __python3 command has version at the end, parse it and expect it.
         # Note that the command is used to determine %python3_sitelib and %python3_sitearch,
         # so we only test known CPython schemes here and not PyPy for simplicity.
@@ -629,7 +674,8 @@ def test_py3_check_import(args, imports, __python3, lib):
         if (match := re.match(r'.+python(\d+\.\d+)$', __python3)):
             x_y = match.group(1)
 
-    lines = rpm_eval(f'%py3_check_import {args}', **macors)
+    invocation = '%{py3_check_import ' + args +'}'
+    lines = rpm_eval(invocation, **macros)
 
     # An equality check is a bit inflexible here,
     # every time we change the macro we need to change this test.
@@ -637,11 +683,31 @@ def test_py3_check_import(args, imports, __python3, lib):
     # At least, let's make the lines saner to check:
     lines = [line.rstrip('\\').strip() for line in lines]
     expected = textwrap.dedent(fr"""
-        (cd TOPDIR &&
         PATH="BUILDROOT/usr/bin:$PATH"
         PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}"
+        _PYTHONSITE="BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages"
         PYTHONDONTWRITEBYTECODE=1
-        {__python3 or '/usr/bin/python3'} -c "import {imports}"
-        )
+        {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {expected_args}
         """)
     assert lines == expected.splitlines()
+
+
+@pytest.mark.parametrize(
+    'shebang_flags_value, expected_shebang_flags',
+    [
+        ('s', '-s'),
+        ('%{nil}', ''),
+        (None, ''),
+        ('Es', '-Es'),
+    ]
+)
+def test_py3_check_import_respects_shebang_flags(shebang_flags_value, expected_shebang_flags, lib):
+    macros = {
+        '_rpmconfigdir': 'RPMCONFIGDIR',
+        '__python3': '/usr/bin/python3',
+        'py3_shebang_flags': shebang_flags_value,
+    }
+    lines = rpm_eval('%py3_check_import sys', **macros)
+    # Compare the last line of the command, that's where lua part is evaluated
+    expected = f'/usr/bin/python3 {expected_shebang_flags} RPMCONFIGDIR/redhat/import_all_modules.py sys'
+    assert  lines[-1].strip() == expected

tests/test_import_all_modules.py

@@ -0,0 +1,426 @@
+from import_all_modules import argparser, exclude_unwanted_module_globs
+from import_all_modules import main as modules_main
+from import_all_modules import read_modules_from_cli, filter_top_level_modules_only
+
+from pathlib import Path
+
+import pytest
+import shlex
+import sys
+
+
+@pytest.fixture(autouse=True)
+def preserve_sys_path():
+    original_sys_path = list(sys.path)
+    yield
+    sys.path = original_sys_path
+
+
+@pytest.fixture(autouse=True)
+def preserve_sys_modules():
+    original_sys_modules = dict(sys.modules)
+    yield
+    sys.modules = original_sys_modules
+
+
+@pytest.mark.parametrize(
+    'args, imports',
+    [
+        ('six', ['six']),
+        ('five  six seven', ['five', 'six', 'seven']),
+        ('six,seven, eight', ['six', 'seven', 'eight']),
+        ('six.quarter  six.half,, SIX', ['six.quarter', 'six.half', 'SIX']),
+        ('six.quarter  six.half,, SIX \\ ', ['six.quarter', 'six.half', 'SIX']),
+    ]
+)
+def test_read_modules_from_cli(args, imports):
+    argv = shlex.split(args)
+    cli_args = argparser().parse_args(argv)
+    assert read_modules_from_cli(cli_args.modules) == imports
+
+
+@pytest.mark.parametrize(
+    'all_mods, imports',
+    [
+        (['six'], ['six']),
+        (['five', 'six', 'seven'], ['five', 'six', 'seven']),
+        (['six.seven', 'eight'], ['eight']),
+        (['SIX', 'six.quarter', 'six.half.and.sth', 'seven'], ['SIX', 'seven']),
+    ],
+)
+def test_filter_top_level_modules_only(all_mods, imports):
+    assert filter_top_level_modules_only(all_mods) == imports
+
+
+@pytest.mark.parametrize(
+    'globs, expected',
+    [
+        (['*.*'], ['foo', 'boo']),
+        (['?oo'], ['foo.bar', 'foo.bar.baz', 'foo.baz']),
+        (['*.baz'], ['foo', 'foo.bar', 'boo']),
+        (['foo'], ['foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']),
+        (['foo*'], ['boo']),
+        (['foo*', '*bar'], ['boo']),
+        (['foo', 'bar'], ['foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']),
+        (['*'], []),
+    ]
+)
+def test_exclude_unwanted_module_globs(globs, expected):
+    my_modules = ['foo', 'foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']
+    tested = exclude_unwanted_module_globs(globs, my_modules)
+    assert tested == expected
+
+
+def test_cli_with_all_args():
+    '''A smoke test, all args must be parsed correctly.'''
+    mods = ['foo', 'foo.bar', 'baz']
+    files = ['-f', './foo']
+    top = ['-t']
+    exclude = ['-e', 'foo*']
+    cli_args = argparser().parse_args([*mods, *files, *top, *exclude])
+
+    assert cli_args.filename == [Path('foo')]
+    assert cli_args.top_level is True
+    assert cli_args.modules == ['foo', 'foo.bar', 'baz']
+    assert cli_args.exclude == ['foo*']
+
+
+def test_cli_without_filename_toplevel():
+    '''Modules provided on command line (without files) must be parsed correctly.'''
+    mods = ['foo', 'foo.bar', 'baz']
+    cli_args = argparser().parse_args(mods)
+
+    assert cli_args.filename is None
+    assert cli_args.top_level is False
+    assert cli_args.modules == ['foo', 'foo.bar', 'baz']
+
+
+def test_cli_with_filename_no_cli_mods():
+    '''Files (without any modules provided on command line) must be parsed correctly.'''
+
+    files = ['-f', './foo', '-f', './bar', '-f', './baz']
+    cli_args = argparser().parse_args(files)
+
+    assert cli_args.filename == [Path('foo'), Path('./bar'), Path('./baz')]
+    assert not cli_args.top_level
+
+
+def test_main_raises_error_when_no_modules_provided():
+    '''If no filename nor modules were provided, ValueError is raised.'''
+
+    with pytest.raises(ValueError):
+        modules_main([])
+
+
+def test_import_all_modules_does_not_import():
+    '''Ensure the files from /usr/lib/rpm/redhat cannot be imported and
+    checked for import'''
+
+    # We already imported it in this file once, make sure it's not imported
+    # from the cache
+    sys.modules.pop('import_all_modules')
+    with pytest.raises(ModuleNotFoundError):
+        modules_main(['import_all_modules'])
+
+
+def test_modules_from_cwd_not_found(tmp_path, monkeypatch):
+    test_module = tmp_path / 'this_is_a_module_in_cwd.py'
+    test_module.write_text('')
+    monkeypatch.chdir(tmp_path)
+    with pytest.raises(ModuleNotFoundError):
+        modules_main(['this_is_a_module_in_cwd'])
+
+
+def test_modules_from_sys_path_found(tmp_path):
+    test_module = tmp_path / 'this_is_a_module_in_sys_path.py'
+    test_module.write_text('')
+    sys.path.append(str(tmp_path))
+    modules_main(['this_is_a_module_in_sys_path'])
+    assert 'this_is_a_module_in_sys_path' in sys.modules
+
+
+def test_modules_from_file_are_found(tmp_path):
+    test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt'
+    test_file.write_text('math\nwave\nsunau\n')
+
+    # Make sure the tested modules are not already in sys.modules
+    for m in ('math', 'wave', 'sunau'):
+        sys.modules.pop(m, None)
+
+    modules_main(['-f', str(test_file)])
+
+    assert 'sunau' in sys.modules
+    assert 'math' in sys.modules
+    assert 'wave' in sys.modules
+
+
+def test_modules_from_files_are_found(tmp_path):
+    test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt'
+    test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt'
+    test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt'
+
+    test_file_1.write_text('math\nwave\n')
+    test_file_2.write_text('sunau\npathlib\n')
+    test_file_3.write_text('logging\nsunau\n')
+
+    # Make sure the tested modules are not already in sys.modules
+    for m in ('math', 'wave', 'sunau', 'pathlib', 'logging'):
+        sys.modules.pop(m, None)
+
+    modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ])
+    for module in ('sunau', 'math', 'wave', 'pathlib', 'logging'):
+        assert module in sys.modules
+
+
+def test_nonexisting_modules_raise_exception_on_import(tmp_path):
+    test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt'
+    test_file.write_text('nonexisting_module\nanother\n')
+    with pytest.raises(ModuleNotFoundError):
+        modules_main(['-f', str(test_file)])
+
+
+def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys):
+
+    # This one is supposed to raise an error
+    cwd_path = tmp_path / 'test_cwd'
+    Path.mkdir(cwd_path)
+    test_module_1 = cwd_path / 'this_is_a_module_in_cwd.py'
+
+    # Nested structure that is supposed to be importable
+    nested_path_1 = tmp_path / 'nested'
+    nested_path_2 = nested_path_1 / 'more_nested'
+
+    for path in (nested_path_1, nested_path_2):
+        Path.mkdir(path)
+
+    test_module_2 = tmp_path / 'this_is_a_module_in_level_0.py'
+    test_module_3 = nested_path_1 / 'this_is_a_module_in_level_1.py'
+    test_module_4 = nested_path_2 / 'this_is_a_module_in_level_2.py'
+
+    for module in (test_module_1, test_module_2, test_module_3, test_module_4):
+        module.write_text('')
+
+    sys.path.append(str(tmp_path))
+    monkeypatch.chdir(cwd_path)
+
+    with pytest.raises(ModuleNotFoundError):
+        modules_main([
+            'this_is_a_module_in_level_0',
+            'nested.this_is_a_module_in_level_1',
+            'nested.more_nested.this_is_a_module_in_level_2',
+            'this_is_a_module_in_cwd'])
+
+    _, err = capsys.readouterr()
+    assert 'Check import: this_is_a_module_in_level_0' in err
+    assert 'Check import: nested.this_is_a_module_in_level_1' in err
+    assert 'Check import: nested.more_nested.this_is_a_module_in_level_2' in err
+    assert 'Check import: this_is_a_module_in_cwd' in err
+
+
+def test_modules_both_from_files_and_cli_are_imported(tmp_path):
+    test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt'
+    test_file_1.write_text('this_is_a_module_in_tmp_path_1')
+
+    test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt'
+    test_file_2.write_text('this_is_a_module_in_tmp_path_2')
+
+    test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py'
+    test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py'
+    test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py'
+
+    for module in (test_module_1, test_module_2, test_module_3):
+        module.write_text('')
+
+    sys.path.append(str(tmp_path))
+    modules_main([
+        '-f', str(test_file_1),
+        'this_is_a_module_in_tmp_path_3',
+        '-f', str(test_file_2),
+    ])
+
+    expected = (
+        'this_is_a_module_in_tmp_path_1',
+        'this_is_a_module_in_tmp_path_2',
+        'this_is_a_module_in_tmp_path_3',
+    )
+    for module in expected:
+        assert module in sys.modules
+
+
+def test_non_existing_module_raises_exception(tmp_path):
+
+    test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py'
+    test_module_1.write_text('')
+    sys.path.append(str(tmp_path))
+
+    with pytest.raises(ModuleNotFoundError):
+        modules_main([
+            'this_is_a_module_in_tmp_path_1',
+            'this_is_a_module_in_tmp_path_2',
+        ])
+
+
+def test_module_with_error_propagates_exception(tmp_path):
+
+    test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py'
+    test_module_1.write_text('0/0')
+    sys.path.append(str(tmp_path))
+
+    # The correct exception must be raised
+    with pytest.raises(ZeroDivisionError):
+        modules_main([
+            'this_is_a_module_in_tmp_path_1',
+        ])
+
+
+def test_correct_modules_are_excluded(tmp_path):
+    test_module_1 = tmp_path / 'module_in_tmp_path_1.py'
+    test_module_2 = tmp_path / 'module_in_tmp_path_2.py'
+    test_module_3 = tmp_path / 'module_in_tmp_path_3.py'
+
+    for module in (test_module_1, test_module_2, test_module_3):
+        module.write_text('')
+
+    sys.path.append(str(tmp_path))
+    test_file_1 = tmp_path / 'a_file_in_tmp_path_1.txt'
+    test_file_1.write_text('module_in_tmp_path_1\nmodule_in_tmp_path_2\nmodule_in_tmp_path_3\n')
+
+    modules_main([
+        '-e', 'module_in_tmp_path_2',
+        '-f', str(test_file_1),
+        '-e', 'module_in_tmp_path_3',
+        ])
+
+    assert 'module_in_tmp_path_1' in sys.modules
+    assert 'module_in_tmp_path_2' not in sys.modules
+    assert 'module_in_tmp_path_3' not in sys.modules
+
+
+def test_excluding_all_modules_raises_error(tmp_path):
+    test_module_1 = tmp_path / 'module_in_tmp_path_1.py'
+    test_module_2 = tmp_path / 'module_in_tmp_path_2.py'
+    test_module_3 = tmp_path / 'module_in_tmp_path_3.py'
+
+    for module in (test_module_1, test_module_2, test_module_3):
+        module.write_text('')
+
+    sys.path.append(str(tmp_path))
+    test_file_1 = tmp_path / 'a_file_in_tmp_path_1.txt'
+    test_file_1.write_text('module_in_tmp_path_1\nmodule_in_tmp_path_2\nmodule_in_tmp_path_3\n')
+
+    with pytest.raises(ValueError):
+        modules_main([
+            '-e', 'module_in_tmp_path*',
+            '-f', str(test_file_1),
+            ])
+
+
+def test_only_toplevel_modules_found(tmp_path):
+
+    # Nested structure that is supposed to be importable
+    nested_path_1 = tmp_path / 'nested'
+    nested_path_2 = nested_path_1 / 'more_nested'
+
+    for path in (nested_path_1, nested_path_2):
+        Path.mkdir(path)
+
+    test_module_1 = tmp_path / 'this_is_a_module_in_level_0.py'
+    test_module_2 = nested_path_1 / 'this_is_a_module_in_level_1.py'
+    test_module_3 = nested_path_2 / 'this_is_a_module_in_level_2.py'
+
+    for module in (test_module_1, test_module_2, test_module_3):
+        module.write_text('')
+
+    sys.path.append(str(tmp_path))
+
+    modules_main([
+        'this_is_a_module_in_level_0',
+        'nested.this_is_a_module_in_level_1',
+        'nested.more_nested.this_is_a_module_in_level_2',
+        '-t'])
+
+    assert 'nested.this_is_a_module_in_level_1' not in sys.modules
+    assert 'nested.more_nested.this_is_a_module_in_level_2' not in sys.modules
+
+
+def test_only_toplevel_included_modules_found(tmp_path):
+
+    # Nested structure that is supposed to be importable
+    nested_path_1 = tmp_path / 'nested'
+    nested_path_2 = nested_path_1 / 'more_nested'
+
+    for path in (nested_path_1, nested_path_2):
+        Path.mkdir(path)
+
+    test_module_1 = tmp_path / 'this_is_a_module_in_level_0.py'
+    test_module_4 = tmp_path / 'this_is_another_module_in_level_0.py'
+
+    test_module_2 = nested_path_1 / 'this_is_a_module_in_level_1.py'
+    test_module_3 = nested_path_2 / 'this_is_a_module_in_level_2.py'
+
+    for module in (test_module_1, test_module_2, test_module_3, test_module_4):
+        module.write_text('')
+
+    sys.path.append(str(tmp_path))
+
+    modules_main([
+        'this_is_a_module_in_level_0',
+        'this_is_another_module_in_level_0',
+        'nested.this_is_a_module_in_level_1',
+        'nested.more_nested.this_is_a_module_in_level_2',
+        '-t',
+        '-e', '*another*'
+    ])
+
+    assert 'nested.this_is_a_module_in_level_1' not in sys.modules
+    assert 'nested.more_nested.this_is_a_module_in_level_2' not in sys.modules
+    assert 'this_is_another_module_in_level_0' not in sys.modules
+    assert 'this_is_a_module_in_level_0' in sys.modules
+
+
+def test_module_list_from_relative_path(tmp_path, monkeypatch):
+
+    monkeypatch.chdir(tmp_path)
+    test_file_1 = Path('this_is_a_file_in_cwd.txt')
+    test_file_1.write_text('wave')
+
+    sys.modules.pop('wave', None)
+
+    modules_main([
+        '-f', 'this_is_a_file_in_cwd.txt'
+    ])
+
+    assert 'wave' in sys.modules
+
+
+@pytest.mark.parametrize('arch_in_path', [True, False])
+def test_pth_files_are_read_from__PYTHONSITE(arch_in_path, tmp_path, monkeypatch, capsys):
+    sitearch = tmp_path / 'lib64'
+    sitearch.mkdir()
+    sitelib = tmp_path / 'lib'
+    sitelib.mkdir()
+
+    for where, word in (sitearch, "ARCH"), (sitelib, "LIB"), (sitelib, "MOD"):
+        module = where / f'print{word}.py'
+        module.write_text(f'print("{word}")')
+
+    pth_sitearch = sitearch / 'ARCH.pth'
+    pth_sitearch.write_text('import printARCH\n')
+
+    pth_sitelib = sitelib / 'LIB.pth'
+    pth_sitelib.write_text('import printLIB\n')
+
+    if arch_in_path:
+        sys.path.append(str(sitearch))
+    sys.path.append(str(sitelib))
+
+    # we always add sitearch to _PYTHONSITE
+    # but when not in sys.path, it should not be processed for .pth files
+    monkeypatch.setenv('_PYTHONSITE', f'{sitearch}:{sitelib}')
+
+    modules_main(['printMOD'])
+    out, err = capsys.readouterr()
+    if arch_in_path:
+        assert out == 'ARCH\nLIB\nMOD\n'
+    else:
+        assert out == 'LIB\nMOD\n'

tests/tests.yml

@@ -15,7 +15,7 @@
     tests:
     - pytest:
         dir: .
-        run: pytest -v
+        run: PYTHONPATH=/usr/lib/rpm/redhat pytest -v
     - manual_byte_compilation:
         dir: .
         run: rpmbuild -ba pythontest.spec