Move Python related BuildRoot Policy scripts from redhat-rpm-config to python-srpm-macros

This allows us to maintain our own BuildRoot Policy scripts in an easier way.

This change needs to be coordinated with the removal of the files from redhat-rpm-config.

redhat-rpm-config requires python-srpm-macros, so no change is expected for the packagers.

brp-fix-pyc-reproducibility

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+# If using normal root, avoid changing anything.
+if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then
+  exit 0
+fi
+
+# Defined as %py_reproducible_pyc_path macro and passed here as
+# the first command-line argument
+path_to_fix=$1
+
+# First, check that the parser is available:
+if [ ! -x /usr/bin/marshalparser ]; then
+  echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !"
+  exit 1
+fi
+
+find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite

brp-python-bytecompile

@@ -0,0 +1,141 @@
+#!/bin/bash
+errors_terminate=$2
+
+# Usage of %_python_bytecompile_extra is not allowed anymore
+# See: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3
+# Therefore $1 ($default_python) is not needed and is invoked with "" by default.
+# $default_python stays in the arguments for backward compatibility and $extra for the following check:
+extra=$3
+if [ 0$extra -eq 1 ]; then
+    echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr
+    exit 1
+fi
+
+# If using normal root, avoid changing anything.
+if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then
+	exit 0
+fi
+
+# Figure out how deep we need to descend.  We could pick an insanely high
+# number and hope it's enough, but somewhere, somebody's sure to run into it.
+depth=`(find "$RPM_BUILD_ROOT" -type f -name "*.py" -print0 ; echo /) | \
+       xargs -0 -n 1 dirname | sed 's,[^/],,g' | sort -u | tail -n 1 | wc -c`
+if [ -z "$depth" -o "$depth" -le "1" ]; then
+	exit 0
+fi
+
+# This function now implements Python byte-compilation in three different ways:
+# Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2
+# Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks
+# When we drop support for Python 2, we'd be able to use all compileall2 features like:
+# - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir
+# - -o 0 -o 1 to produce multiple files in one run - each with a different optimization level - instead of $options
+# - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit()
+# These changes will make this script much simpler
+# In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again.
+function python_bytecompile()
+{
+    local options=$1
+    local python_binary=$2
+    local exclude=$3
+    local python_libdir="$4"
+    local depth=$5 # Not used for Python >= 3.4
+    local real_libdir=$6 # Not used for Python >= 3.4
+
+	python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
+
+	#
+	# Python 3.9 and higher
+	#
+	if [ "$python_version" -ge 39 ]; then
+
+		[ ! -z $exclude ] && exclude="-x '$exclude'"
+		# -q disables verbose output
+		# -f forces the process to overwrite existing compiled files
+		# -x excludes paths defined by regex
+		# -e excludes symbolic links pointing outside the build root
+		# -x and -e together implements the same functionality as the Filter class below
+		# -s strips $RPM_BUILD_ROOT from the path
+		# -p prepends the leading slash to the path to make it absolute
+		$python_binary -B $options -m compileall -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir"
+
+	#
+	# Python 3.4 and higher
+	#
+	elif [ "$python_version" -ge 34 ]; then
+
+		[ ! -z $exclude ] && exclude="-x '$exclude'"
+		# /usr/lib/rpm/redhat/ contains compileall2 Python module
+		# -q disables verbose output
+		# -f forces the process to overwrite existing compiled files
+		# -x excludes paths defined by regex
+		# -e excludes symbolic links pointing outside the build root
+		# -x and -e together implements the same functionality as the Filter class below
+		# -s strips $RPM_BUILD_ROOT from the path
+		# -p prepends the leading slash to the path to make it absolute
+		PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B $options -m compileall2 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir"
+	else
+#
+# Python 3.3 and lower (incl. Python 2)
+#
+
+cat << EOF | $python_binary $options
+import compileall, sys, os, re
+
+python_libdir = "$python_libdir"
+depth = $depth
+real_libdir = "$real_libdir"
+build_root = "$RPM_BUILD_ROOT"
+exclude = r"$exclude"
+
+class Filter:
+    def search(self, path):
+        ret = not os.path.realpath(path).startswith(build_root)
+        if exclude:
+            ret = ret or re.search(exclude, path)
+        return ret
+
+sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1))
+EOF
+
+fi
+}
+
+# .pyc/.pyo files embed a "magic" value, identifying the ABI version of Python
+# bytecode that they are for.
+#
+# The files below RPM_BUILD_ROOT could be targeting multiple versions of
+# python (e.g. a single build that emits several subpackages e.g. a
+# python26-foo subpackage, a python31-foo subpackage etc)
+#
+# Support this by assuming that below each /usr/lib/python$VERSION/, all
+# .pyc/.pyo files are to be compiled for /usr/bin/python$VERSION.
+#
+# For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6
+# and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1
+
+# Disable Python hash seed randomization
+# This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078
+export PYTHONHASHSEED=0
+
+shopt -s nullglob
+find "$RPM_BUILD_ROOT" -type d -print0|grep -z -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$" | while read -d "" python_libdir;
+do
+	python_binary=$(basename "$python_libdir")
+	real_libdir=${python_libdir/$RPM_BUILD_ROOT/}
+	echo "Bytecompiling .py files below $python_libdir using $python_binary"
+
+	# Generate normal (.pyc) byte-compiled files.
+	python_bytecompile "" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir"
+	if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then
+		# One or more of the files had a syntax error
+		exit 1
+	fi
+
+	# Generate optimized (.pyo) byte-compiled files.
+	python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir"
+	if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then
+		# One or more of the files had a syntax error
+		exit 1
+	fi
+done

macros.python-srpm

@@ -50,9 +50,32 @@
 %python3_pkgversion 3
 
 
-# BRP scripts, they need to be included in redhat-rpm-macros, in %%__os_install_post
+### BRP scripts (and related macros)
+
+## Automatically compile python files
+%py_auto_byte_compile 1
+## Should python bytecompilation errors terminate a build?
+%_python_bytecompile_errors_terminate_build 1
+## Should python bytecompilation compile outside python specific directories?
+## This always causes errors when enabled, see https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3
+%_python_bytecompile_extra 0
+
+## The individual BRP scripts
+%__brp_python_bytecompile %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}"
+%__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility
 %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink
 
+## This macro is included in redhat-rpm-config's %%__os_install_post
+# Note that the order matters:
+#  1. brp-python-bytecompile can create (or replace) pyc files
+#  2. brp-fix-pyc-reproducibility can modify the pyc files from above
+#  3. brp-python-hardlink de-duplicates identical pyc files
+%__os_install_post_python \
+    %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \
+    %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \
+    %{?__brp_python_hardlink} \
+%{nil}
+
 
 # === Macros for Build/Requires tags using Python dist tags ===
 # - https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages

python-rpm-macros.spec

@@ -17,14 +17,22 @@ Source201:      python.lua
 Source301:      https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py
 
 # BRP scripts
+# This one is from redhat-rpm-config < 190
+# A new upstream is forming in https://github.com/rpm-software-management/python-rpm-packaging/blob/main/scripts/brp-python-bytecompile
+# But our version is riddled with Fedora-isms
+# We might eventually move to upstream source + Fedora patches, but we are not there yet
+Source401:      brp-python-bytecompile
 # This one is from https://github.com/rpm-software-management/python-rpm-packaging/blob/main/scripts/brp-python-hardlink
 # But we don't use a link in case it changes in upstream, there are no "versions" there yet
 # This was removed from RPM 4.17+ so we maintain it here instead
-Source401:      brp-python-hardlink
+Source402:      brp-python-hardlink
+# This one is from redhat-rpm-config < 190
+# It has no upstream yet
+Source403:      brp-fix-pyc-reproducibility
 
 # macros and lua: MIT
 # compileall2.py: PSFv2
-# brp-python-hardlink: GPLv2+
+# brp scripts: GPLv2+
 License:        MIT and Python and GPLv2+
 
 # The package version MUST be always the same as %%{__default_python3_version}.
@@ -39,7 +47,7 @@ elseif posix.stat('macros.python-srpm') then
 end
 }
 Version:        %{__default_python3_version}
-Release:        5%{?dist}
+Release:        6%{?dist}
 
 BuildArch:      noarch
 
@@ -60,7 +68,8 @@ python?-devel packages require it. So install a python-devel package instead.
 Summary:        RPM macros for building Python source packages
 
 # For directory structure and flags macros
-Requires:       redhat-rpm-config
+# Versions before 190 contained some brp scripts moved into python-srpm-macros
+Requires:       redhat-rpm-config >= 190
 
 # We bundle our own software here :/
 Provides:       bundled(python3dist(compileall2)) = %{compileall2_version}
@@ -105,7 +114,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/
 # It also ensures that:
 #  - our BRPs can execute
 #  - if our BRPs affect this package, we don't need to build it twice
+%global __brp_python_bytecompile %{buildroot}%{__brp_python_bytecompile}
 %global __brp_python_hardlink %{buildroot}%{__brp_python_hardlink}
+%global __brp_fix_pyc_reproducibility %{buildroot}%{__brp_fix_pyc_reproducibility}
 
 
 %check
@@ -120,7 +131,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/
 %files -n python-srpm-macros
 %{rpmmacrodir}/macros.python-srpm
 %{_rpmconfigdir}/redhat/compileall2.py
+%{_rpmconfigdir}/redhat/brp-python-bytecompile
 %{_rpmconfigdir}/redhat/brp-python-hardlink
+%{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility
 %{_rpmluadir}/fedora/srpm/python.lua
 
 %files -n python3-rpm-macros
@@ -128,6 +141,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/
 
 
 %changelog
+* Wed Jul 07 2021 Miro Hrončok <mhroncok@redhat.com> - 3.10-6
+- Move Python related BuildRoot Policy scripts from redhat-rpm-config to python-srpm-macros
+
 * Wed Jul 07 2021 Miro Hrončok <mhroncok@redhat.com> - 3.10-5
 - Introduce %%py3_check_import