From 74e11b4fa8eb7675c3a4b9c5161a1f32cafd4835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 30 Jun 2021 19:25:29 +0200 Subject: [PATCH] Move Python related BuildRoot Policy scripts from redhat-rpm-config to python-srpm-macros This allows us to maintain our own BuildRoot Policy scripts in an easier way. This change needs to be coordinated with the addition of the files to python-srpm-macros. redhat-rpm-config requires python-srpm-macros, so no significant change is expected for the packagers. This also moves the Python BRPs to the end of the list which should be fine. --- brp-fix-pyc-reproducibility | 18 ----- brp-python-bytecompile | 141 ------------------------------------ macros | 20 ++--- redhat-rpm-config.spec | 14 ++-- 4 files changed, 10 insertions(+), 183 deletions(-) delete mode 100644 brp-fix-pyc-reproducibility delete mode 100755 brp-python-bytecompile diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility deleted file mode 100644 index 4118d97..0000000 --- a/brp-fix-pyc-reproducibility +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -# If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then - exit 0 -fi - -# Defined as %py_reproducible_pyc_path macro and passed here as -# the first command-line argument -path_to_fix=$1 - -# First, check that the parser is available: -if [ ! -x /usr/bin/marshalparser ]; then - echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" - exit 1 -fi - -find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite diff --git a/brp-python-bytecompile b/brp-python-bytecompile deleted file mode 100755 index 71f4d2d..0000000 --- a/brp-python-bytecompile +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -errors_terminate=$2 - -# Usage of %_python_bytecompile_extra is not allowed anymore -# See: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3 -# Therefore $1 ($default_python) is not needed and is invoked with "" by default. -# $default_python stays in the arguments for backward compatibility and $extra for the following check: -extra=$3 -if [ 0$extra -eq 1 ]; then - echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr - exit 1 -fi - -# If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then - exit 0 -fi - -# Figure out how deep we need to descend. We could pick an insanely high -# number and hope it's enough, but somewhere, somebody's sure to run into it. -depth=`(find "$RPM_BUILD_ROOT" -type f -name "*.py" -print0 ; echo /) | \ - xargs -0 -n 1 dirname | sed 's,[^/],,g' | sort -u | tail -n 1 | wc -c` -if [ -z "$depth" -o "$depth" -le "1" ]; then - exit 0 -fi - -# This function now implements Python byte-compilation in three different ways: -# Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 -# Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks -# When we drop support for Python 2, we'd be able to use all compileall2 features like: -# - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir -# - -o 0 -o 1 to produce multiple files in one run - each with a different optimization level - instead of $options -# - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit() -# These changes will make this script much simpler -# In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. -function python_bytecompile() -{ - local options=$1 - local python_binary=$2 - local exclude=$3 - local python_libdir="$4" - local depth=$5 # Not used for Python >= 3.4 - local real_libdir=$6 # Not used for Python >= 3.4 - - python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") - - # - # Python 3.9 and higher - # - if [ "$python_version" -ge 39 ]; then - - [ ! -z $exclude ] && exclude="-x '$exclude'" - # -q disables verbose output - # -f forces the process to overwrite existing compiled files - # -x excludes paths defined by regex - # -e excludes symbolic links pointing outside the build root - # -x and -e together implements the same functionality as the Filter class below - # -s strips $RPM_BUILD_ROOT from the path - # -p prepends the leading slash to the path to make it absolute - $python_binary -B $options -m compileall -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir" - - # - # Python 3.4 and higher - # - elif [ "$python_version" -ge 34 ]; then - - [ ! -z $exclude ] && exclude="-x '$exclude'" - # /usr/lib/rpm/redhat/ contains compileall2 Python module - # -q disables verbose output - # -f forces the process to overwrite existing compiled files - # -x excludes paths defined by regex - # -e excludes symbolic links pointing outside the build root - # -x and -e together implements the same functionality as the Filter class below - # -s strips $RPM_BUILD_ROOT from the path - # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B $options -m compileall2 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir" - else -# -# Python 3.3 and lower (incl. Python 2) -# - -cat << EOF | $python_binary $options -import compileall, sys, os, re - -python_libdir = "$python_libdir" -depth = $depth -real_libdir = "$real_libdir" -build_root = "$RPM_BUILD_ROOT" -exclude = r"$exclude" - -class Filter: - def search(self, path): - ret = not os.path.realpath(path).startswith(build_root) - if exclude: - ret = ret or re.search(exclude, path) - return ret - -sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) -EOF - -fi -} - -# .pyc/.pyo files embed a "magic" value, identifying the ABI version of Python -# bytecode that they are for. -# -# The files below RPM_BUILD_ROOT could be targeting multiple versions of -# python (e.g. a single build that emits several subpackages e.g. a -# python26-foo subpackage, a python31-foo subpackage etc) -# -# Support this by assuming that below each /usr/lib/python$VERSION/, all -# .pyc/.pyo files are to be compiled for /usr/bin/python$VERSION. -# -# For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6 -# and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1 - -# Disable Python hash seed randomization -# This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 -export PYTHONHASHSEED=0 - -shopt -s nullglob -find "$RPM_BUILD_ROOT" -type d -print0|grep -z -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$" | while read -d "" python_libdir; -do - python_binary=$(basename "$python_libdir") - real_libdir=${python_libdir/$RPM_BUILD_ROOT/} - echo "Bytecompiling .py files below $python_libdir using $python_binary" - - # Generate normal (.pyc) byte-compiled files. - python_bytecompile "" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then - # One or more of the files had a syntax error - exit 1 - fi - - # Generate optimized (.pyo) byte-compiled files. - python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then - # One or more of the files had a syntax error - exit 1 - fi -done diff --git a/macros b/macros index 2e3072d..365c65d 100644 --- a/macros +++ b/macros @@ -244,8 +244,6 @@ print(result) %__brp_strip_lto /usr/lib/rpm/redhat/brp-strip-lto %{__strip} %__brp_strip_comment_note /usr/lib/rpm/brp-strip-comment-note %{__strip} %{__objdump} %__brp_strip_static_archive /usr/lib/rpm/brp-strip-static-archive %{__strip} -%__brp_python_bytecompile /usr/lib/rpm/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" -%__brp_fix_pyc_reproducibility /usr/lib/rpm/redhat/brp-fix-pyc-reproducibility %__brp_check_rpaths /usr/lib/rpm/check-rpaths # __brp_mangle_shebangs_exclude - shebangs to exclude # __brp_mangle_shebangs_exclude_file - file from which to get shebangs to exclude @@ -255,6 +253,10 @@ print(result) %__brp_llvm_compile_lto_elf /usr/lib/rpm/redhat/brp-llvm-compile-lto-elf %{build_cflags} %{build_ldflags} +# note: %%__os_install_post_python is defined in python-srpm-macros and contains several policies +# redhat-rpm-config maintainers, don't remove it from %%__os_install_post unless coordinating the change with Python maintainers +# packagers, don't undefine the entire macro, see the individual macros in /usr/lib/rpm/macros.d/macros.python-srpm + %__os_install_post \ %[ "%{toolchain}" == "clang" ? "%{?__brp_llvm_compile_lto_elf}" : "%{nil}" ] \ %{?__brp_ldconfig} \ @@ -265,11 +267,9 @@ print(result) } \ %{?__brp_strip_lto} \ %{?__brp_strip_static_archive} \ - %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ - %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ %{?__brp_check_rpaths} \ - %{?__brp_python_hardlink} \ %{?__brp_mangle_shebangs} \ + %{?__os_install_post_python} \ %{nil} %__spec_install_post\ @@ -286,16 +286,6 @@ print(result) # Should missing buildids terminate a build? %_missing_build_ids_terminate_build 1 -# -## Automatically compile python files -%py_auto_byte_compile 1 - -# -## Should python bytecompilation errors terminate a build? -%_python_bytecompile_errors_terminate_build 1 -## Should python bytecompilation compile outisde python specific directories? -%_python_bytecompile_extra 0 - # Use SHA-256 for FILEDIGESTS instead of default MD5 %_source_filedigest_algorithm 8 %_binary_filedigest_algorithm 8 diff --git a/redhat-rpm-config.spec b/redhat-rpm-config.spec index e95a2e6..eab9248 100644 --- a/redhat-rpm-config.spec +++ b/redhat-rpm-config.spec @@ -6,7 +6,7 @@ Summary: Red Hat specific rpm configuration files Name: redhat-rpm-config -Version: 189 +Version: 190 Release: 1%{?dist} # No version specified. License: GPL+ @@ -49,14 +49,6 @@ Source155: macros.fedora-misc # and an echo when the mangling happens Source201: brp-mangle-shebangs -# this comes from rpm itself -# however, now we can do Fedora changes within -Source202: brp-python-bytecompile - -# for fixing pyc files reproducibility with marshalparser -# https://github.com/fedora-python/marshalparser -Source203: brp-fix-pyc-reproducibility - # for converting llvm LTO bitcode objects into ELF Source204: brp-llvm-compile-lto-elf @@ -111,6 +103,7 @@ Requires: ocaml-srpm-macros Requires: openblas-srpm-macros Requires: perl-srpm-macros # ↓ Provides compileall2 Python module +# TODO: require >= 3.10-6 in the next iteration to ensure Python BRPs are always present Requires: python-srpm-macros >= 3-46 Requires: qt5-srpm-macros Requires: rust-srpm-macros @@ -221,6 +214,9 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm forge.lua %{_rpmconfigdir}/macros.d/macros.kmp %changelog +* Wed Jun 30 2021 Miro Hrončok - 190-1 +- Move Python related BuildRoot Policy scripts from redhat-rpm-config to python-srpm-macros + * Mon Jun 28 2021 Ben Burton - 189-1 - Adapt macros and BRP scripts for %%topdir with spaces - Fixes rhbz#1947416