From 5f32aac09e20fa00dd6b573ada95bf363591e8c8 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 16 Jul 2020 10:30:55 +0200 Subject: [PATCH] New opt-in possibility to fix byte-compilation reproducibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new script brp-fix-pyc-reproducibility creates an opt-in way of how to fix problems with the reproducibility of byte-compiled Python files. The script uses marshalparser [0] which currently doesn't provide solutions for all issues but can fix at least problems with reference flags. For more info see this Bugzilla [1]. If you want to use this new feature, you need to define `%py_reproducible_pyc_path` to specify a path you want to fix `.pyc` files in (recursively) and build-require /usr/bin/marshalparser. if you forget to build-require the parser. The error message is: ``` + /usr/lib/rpm/redhat/brp-python-bytecompile '' 1 0 Bytecompiling .py files below /builddir/build/BUILDROOT/tldr-0.5-2.fc33.x86_64/usr/lib/python3.9 using /usr/bin/python3.9 + /usr/lib/rpm/redhat/brp-fix-pyc-reproducibility /builddir/build/BUILDROOT/tldr-0.5-2.fc33.x86_64 ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser ! error: Bad exit status from /var/tmp/rpm-tmp.UUJr4v (%install) ``` A build fails if the parser is not able to parse any of the `.pyc` files. And finally, if a build is properly configured it produces fixed `.pyc` files. Currently, `.pyc` files in the tldr package contain a lot of unused reference flags: ``` $ dnf install -y tldr $ marshalparser --unused /usr/lib/python3.9/site-packages/__pycache__/tldr.cpython-39.pyc … long output … 190 - Flag_ref(byte=9610, type='TYPE_SHORT_ASCII_INTERNED', content=b'init', usages=0) 191 - Flag_ref(byte=9633, type='TYPE_SHORT_ASCII_INTERNED', content=b'source', usages=0) 192 - Flag_ref(byte=9651, type='TYPE_SHORT_ASCII_INTERNED', content=b'argv', usages=0) 193 - Flag_ref(byte=9657, type='TYPE_SHORT_ASCII_INTERNED', content=b'print_help', usages=0) 194 - Flag_ref(byte=9669, type='TYPE_SHORT_ASCII_INTERNED', content=b'stderr', usages=0) 195 - Flag_ref(byte=9682, type='TYPE_SHORT_ASCII_INTERNED', content=b'parse_args', usages=0) 196 - Flag_ref(byte=9737, type='TYPE_SHORT_ASCII_INTERNED', content=b'encode', usages=0) 197 - Flag_ref(byte=9782, type='TYPE_SHORT_ASCII_INTERNED', content=b'parser', usages=0) 198 - Flag_ref(byte=9790, type='TYPE_SHORT_ASCII_INTERNED', content=b'options', usages=0) 199 - Flag_ref(byte=9799, type='TYPE_SHORT_ASCII_INTERNED', content=b'rest', usages=0) 200 - Flag_ref(byte=9821, type='TYPE_SHORT_ASCII_INTERNED', content=b'result', usages=0) 202 - Flag_ref(byte=10022, type='TYPE_SHORT_ASCII_INTERNED', content=b'__main__', usages=0) 203 - Flag_ref(byte=10102, type='TYPE_SHORT_ASCII_INTERNED', content=b'argparse', usages=0) 204 - Flag_ref(byte=10433, type='TYPE_SHORT_ASCII_INTERNED', content=b'__name__', usages=0) 205 - Flag_ref(byte=10463, type='TYPE_SHORT_ASCII_INTERNED', content=b'', usages=0) ``` This new feature fixes them: ``` $ marshalparser --unused /usr/lib/python3.9/site-packages/__pycache__/tldr.cpython-39.pyc ``` [0] https://github.com/fedora-python/marshalparser [1] https://bugzilla.redhat.com/show_bug.cgi?id=1686078 --- brp-fix-pyc-reproducibility | 18 ++++++++++++++++++ macros | 2 ++ redhat-rpm-config.spec | 11 +++++++++-- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 brp-fix-pyc-reproducibility diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility new file mode 100644 index 0000000..4118d97 --- /dev/null +++ b/brp-fix-pyc-reproducibility @@ -0,0 +1,18 @@ +#!/bin/bash -e + +# If using normal root, avoid changing anything. +if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then + exit 0 +fi + +# Defined as %py_reproducible_pyc_path macro and passed here as +# the first command-line argument +path_to_fix=$1 + +# First, check that the parser is available: +if [ ! -x /usr/bin/marshalparser ]; then + echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" + exit 1 +fi + +find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite diff --git a/macros b/macros index 6bf915f..20396fb 100644 --- a/macros +++ b/macros @@ -233,6 +233,7 @@ print(result) %__brp_strip_comment_note /usr/lib/rpm/brp-strip-comment-note %{__strip} %{__objdump} %__brp_strip_static_archive /usr/lib/rpm/brp-strip-static-archive %{__strip} %__brp_python_bytecompile /usr/lib/rpm/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" +%__brp_fix_pyc_reproducibility /usr/lib/rpm/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink /usr/lib/rpm/brp-python-hardlink # __brp_mangle_shebangs_exclude - shebangs to exclude # __brp_mangle_shebangs_exclude_file - file from which to get shebangs to exclude @@ -250,6 +251,7 @@ print(result) %{?__brp_strip_lto} \ %{?__brp_strip_static_archive} \ %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ + %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ %{?__brp_python_hardlink} \ %{?__brp_mangle_shebangs} \ %{nil} diff --git a/redhat-rpm-config.spec b/redhat-rpm-config.spec index 2a657a4..7cd0eaa 100644 --- a/redhat-rpm-config.spec +++ b/redhat-rpm-config.spec @@ -6,8 +6,8 @@ Summary: Red Hat specific rpm configuration files Name: redhat-rpm-config -Version: 161 -Release: 2%{?dist} +Version: 162 +Release: 1%{?dist} # No version specified. License: GPL+ URL: https://src.fedoraproject.org/rpms/redhat-rpm-config @@ -53,6 +53,10 @@ Source201: brp-mangle-shebangs # however, now we can do Fedora changes within Source202: brp-python-bytecompile +# for fixing pyc files reproducibility with marshalparser +# https://github.com/fedora-python/marshalparser +Source203: brp-fix-pyc-reproducibility + # Dependency generator scripts (deprecated) Source300: find-provides Source301: find-provides.ksyms @@ -207,6 +211,9 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm forge.lua %{_rpmconfigdir}/macros.d/macros.kmp %changelog +* Thu Jul 16 2020 Lumír Balhar - 162-1 +- New script brp-fix-pyc-reproducibility + * Tue Jun 16 2020 Lumír Balhar - 161-2 - Use stdlib compileall for Python >= 3.9