diff --git a/brp-mangle-shebangs b/brp-mangle-shebangs index 67a1a7d..98990c0 100755 --- a/brp-mangle-shebangs +++ b/brp-mangle-shebangs @@ -70,13 +70,17 @@ done cd "$RPM_BUILD_ROOT" -trim() { - printf '%s' "$*" -} - +# Large packages such as kernel can have thousands of executable files. +# We take care to not fork/exec thousands of "file"s and "grep"s, +# but run just two of them. +# (Take care to exclude filenames which would mangle "file" output). +find -executable -type f ! -path '*:*' ! -path $'*\n*' \ +| file -N --mime-type -f - \ +| grep -P ".+(?=: text/)" \ +| { fail=0 -while IFS= read -r -d $'\0' f; do - file -N --mime-type "$f" | grep -q -P ".+(?=: text/)" || continue +while IFS= read -r line; do + f=${line%%:*} # Remove the dot path="${f#.}" @@ -88,24 +92,34 @@ while IFS= read -r -d $'\0' f; do echo "$path" | grep -q -E -f "$exclude_files_from" && continue fi - ts=$(stat -c %y "$f") - read shebang_line < "$f" || : - orig_shebang=$(trim $(echo "$shebang_line" | grep -Po "#!\K.*" || echo)) - shebang="$orig_shebang" - if [ -n "$exclude_shebangs" ]; then - echo "$shebang" | grep -q -E "$exclude_shebangs" && continue - fi - if [ -n "$exclude_shebangs_from" ]; then - echo "$shebang" | grep -q -E -f "$exclude_shebangs_from" && continue - fi - - if [ -z "$shebang" ]; then - echo >&2 "*** WARNING: $f is executable but has empty or no shebang, removing executable bit" + read shebang_line < "$f" + orig_shebang="${shebang_line#\#!}" + if [ "$orig_shebang" = "$shebang_line" ]; then + echo >&2 "*** WARNING: $f is executable but has no shebang, removing executable bit" + ts=$(stat -c %y "$f") chmod -x "$f" touch -d "$ts" "$f" continue - elif [ -n "${shebang##/*}" ]; then + fi + + # Trim spaces + while shebang="${orig_shebang// / }"; [ "$shebang" != "$orig_shebang" ]; do + orig_shebang="$shebang" + done + # Treat "#! /path/to " as "#!/path/to" + orig_shebang="${orig_shebang# }" + + shebang="$orig_shebang" + + if [ -z "$shebang" ]; then + echo >&2 "*** WARNING: $f is executable but has empty shebang, removing executable bit" + ts=$(stat -c %y "$f") + chmod -x "$f" + touch -d "$ts" "$f" + continue + fi + if [ -n "${shebang##/*}" ]; then echo >&2 "*** ERROR: $f has shebang which doesn't start with '/' ($shebang)" fail=1 continue @@ -132,11 +146,13 @@ while IFS= read -r -d $'\0' f; do echo >&2 "*** ERROR: ambiguous python shebang in $path: #!$orig_shebang. Change it to python3 (or python2) explicitly." fail=1 elif [ "#!$shebang" != "#!$orig_shebang" ]; then - sed -i -e "1c #!$shebang" "$f" echo "mangling shebang in $path from $orig_shebang to #!$shebang" + ts=$(stat -c %y "$f") + sed -i -e "1c #!$shebang" "$f" + touch -d "$ts" "$f" fi - touch -d "$ts" "$f" -done < <(find -executable -type f -print0) +done exit $fail +} diff --git a/brp-python-bytecompile b/brp-python-bytecompile new file mode 100755 index 0000000..4727feb --- /dev/null +++ b/brp-python-bytecompile @@ -0,0 +1,144 @@ +#!/bin/bash +errors_terminate=$2 +extra=$3 + +# If using normal root, avoid changing anything. +if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then + exit 0 +fi + +# Figure out how deep we need to descend. We could pick an insanely high +# number and hope it's enough, but somewhere, somebody's sure to run into it. +depth=`(find "$RPM_BUILD_ROOT" -type f -name "*.py" -print0 ; echo /) | \ + xargs -0 -n 1 dirname | sed 's,[^/],,g' | sort -u | tail -n 1 | wc -c` +if [ -z "$depth" -o "$depth" -le "1" ]; then + exit 0 +fi + +# This function now implements Python byte-compilation in two different ways: +# Python >= 3.4 uses a new module compileall2 - https://github.com/fedora-python/compileall2 +# Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks +# When we drop support for Python 2, we'd be able to use all compileall2 features like: +# - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir +# - -o 0 -o 1 to produce multiple files in one run - each with a different optimization level - instead of $options +# - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit() +# These changes will make this script much simpler +function python_bytecompile() +{ + local options=$1 + local python_binary=$2 + local exclude=$3 + local python_libdir=$4 + local depth=$5 # Not used for Python >= 3.4 + local real_libdir=$6 # Not used for Python >= 3.4 + + python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") + + # + # Python 3.4 and higher + # + if [ "$python_version" -ge 34 ]; then + + [ ! -z $exclude ] && exclude="-x '$exclude'" + # /usr/lib/rpm/redhat/ contains compileall2 Python module + # -q disables verbose output + # -f forces the process to overwrite existing compiled files + # -x excludes paths defined by regex + # -e excludes symbolic links pointing outside the build root + # -x and -e together implements the same functionality as the Filter class below + # -s strips $RPM_BUILD_ROOT from the path + # -p prepends the leading slash to the path to make it absolute + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B $options -m compileall2 -q -f $exclude -s $RPM_BUILD_ROOT -p / -e $RPM_BUILD_ROOT $python_libdir + else +# +# Python 3.3 and lower (incl. Python 2) +# + +cat << EOF | $python_binary $options +import compileall, sys, os, re + +python_libdir = "$python_libdir" +depth = $depth +real_libdir = "$real_libdir" +build_root = "$RPM_BUILD_ROOT" +exclude = r"$exclude" + +class Filter: + def search(self, path): + ret = not os.path.realpath(path).startswith(build_root) + if exclude: + ret = ret or re.search(exclude, path) + return ret + +sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) +EOF + +fi +} + +# .pyc/.pyo files embed a "magic" value, identifying the ABI version of Python +# bytecode that they are for. +# +# The files below RPM_BUILD_ROOT could be targeting multiple versions of +# python (e.g. a single build that emits several subpackages e.g. a +# python26-foo subpackage, a python31-foo subpackage etc) +# +# Support this by assuming that below each /usr/lib/python$VERSION/, all +# .pyc/.pyo files are to be compiled for /usr/bin/python$VERSION. +# +# For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6 +# and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1 + +shopt -s nullglob +for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/usr/lib(64)?/python[0-9]\.[0-9]+$"`; +do + python_binary=/usr/bin/$(basename $python_libdir) + real_libdir=${python_libdir/$RPM_BUILD_ROOT/} + echo "Bytecompiling .py files below $python_libdir using $python_binary" + + # Generate normal (.pyc) byte-compiled files. + python_bytecompile "" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 + fi + + # Generate optimized (.pyo) byte-compiled files. + python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 + fi +done + + +# Handle other locations in the filesystem using the default python implementation +# if extra is set to 0, don't do this +if [ 0$extra -eq 0 ]; then + exit 0 +fi + +# If we don't have a default python interpreter, we cannot proceed +default_python=${1:-/usr/bin/python} +if [ ! -x "$default_python" ]; then + exit 0 +fi + +# Figure out if there are files to be bytecompiled with the default_python at all +# this prevents unnecessary default_python invocation +find "$RPM_BUILD_ROOT" -type f -name "*.py" | grep -Ev "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" || exit 0 + +# Generate normal (.pyc) byte-compiled files. +python_bytecompile "" $default_python "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" "$RPM_BUILD_ROOT" "$depth" "/" +if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 +fi + +# Generate optimized (.pyo) byte-compiled files. +python_bytecompile "-O" $default_python "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" "$RPM_BUILD_ROOT" "$depth" "/" +if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 +fi +exit 0 diff --git a/brp-strip-lto b/brp-strip-lto new file mode 100755 index 0000000..8890541 --- /dev/null +++ b/brp-strip-lto @@ -0,0 +1,17 @@ +#!/usr/bin/sh +# If using normal root, avoid changing anything. +if [ -z "$RPM_BUILD_ROOT" ] || [ "$RPM_BUILD_ROOT" = "/" ]; then + exit 0 +fi + +STRIP=${1:-strip} +NCPUS=${RPM_BUILD_NCPUS:-1} + +case `uname -a` in +Darwin*) exit 0 ;; +*) ;; +esac + +# Strip ELF binaries +find "$RPM_BUILD_ROOT" -type f -name '*.[ao]' \! -regex "$RPM_BUILD_ROOT/*usr/lib/debug.*" -print0 | \ + eu-elfclassify --not-program --not-library --not-linux-kernel-module --stdin0 --print0 | xargs -0 -r -P$NCPUS -n32 sh -c "$STRIP -p -R .gnu.lto_* -R .gnu.debuglto_* -N __gnu_lto_v1 \"\$@\"" ARG0 diff --git a/buildflags.md b/buildflags.md index 01ba183..033c070 100644 --- a/buildflags.md +++ b/buildflags.md @@ -15,9 +15,9 @@ This will invoke the `./configure` with arguments (such as `--prefix=/usr`) to adjust the paths to the packaging defaults. As a side effect, this will set the environment variables `CFLAGS`, -`CXXFLAGS`, `FFLAGS`, `FCFLAGS`, and `LDFLAGS`, so they can be used by -makefiles and other build tools. (However, existing values for this -variables are not overwritten.) +`CXXFLAGS`, `FFLAGS`, `FCFLAGS`, `LDFLAGS` and `LT_SYS_LIBRARY_PATH`, +so they can be used by makefiles and other build tools. (However, +existing values for these variables are not overwritten.) If your package does not use autoconf, you can still set the same environment variables using @@ -43,6 +43,9 @@ Individual build flags are also available through RPM macros: driver. At the start of the `%build` section, the environment variable `RPM_LD_FLAGS` is set to this value. +The variable `LT_SYS_LIBRARY_PATH` is defined here to prevent the `libtool` +script (v2.4.6+) from hardcoding %_libdir into the binaries' RPATH. + These RPM macros do not alter shell environment variables. For some other build tools separate mechanisms exist: @@ -145,6 +148,18 @@ to the RPM spec file to disable these strict checks. Alternatively, you can pass `-z undefs` to ld (written as `-Wl,-z,undefs` on the gcc command line). The latter needs binutils 2.29.1-12.fc28 or later. +### Legacy -fcommon + +Since version 10, [gcc defaults to `-fno-common`](https://gcc.gnu.org/gcc-10/porting_to.html#common). +Builds may fail with `multiple definition of ...` errors. + +As a short term workaround for such failure, +it is possible to add `-fcommon` to the flags by defining `%_legacy_common_support`. + + %define _legacy_common_support 1 + +Properly fixing the failure is always preferred! + # Individual compiler flags Compiler flags end up in the environment variables `CFLAGS`, diff --git a/common.lua b/common.lua index e087a1c..e469ba0 100644 --- a/common.lua +++ b/common.lua @@ -138,6 +138,7 @@ end local function wordwrap(text) text = rpm.expand(text .. "\n") text = string.gsub(text, "\t", " ") + text = string.gsub(text, "\r", "\n") text = string.gsub(text, " +\n", "\n") text = string.gsub(text, "\n+\n", "\n\n") text = string.gsub(text, "^\n", "") @@ -160,7 +161,8 @@ local function wordwrap(text) end advance = string.gsub(advance, "– ", " ") pos = pos + wl - elseif (pos + wl < 81) then + elseif (pos + wl < 81) or + ((pos + wl == 81) and string.match(word, "\n$")) then pos = pos + wl else word = advance .. string.gsub(word, "^%s*", "") diff --git a/kmod.prov b/kmod.prov index f02d8a0..bbe5aa0 100644 --- a/kmod.prov +++ b/kmod.prov @@ -1,17 +1,28 @@ #!/bin/sh +x +# Kernel build can have many thousands of modules. +# kmod.prov is run for every one of them. +# Try to make this script run as fast as we can. +# For example, use shell string ops instead of external programs +# where possible. IFS=$'\n' -for i in $(grep -E '(/lib/modules/.*\.ko|/lib/modules/.*/modules.builtin)'); -do - kmod=$(basename $i | sed -e 's/.[xg]z//'); +read -r fname || exit - if [ $kmod == "modules.builtin" ]; then - for j in $(cat $i); do - j=$(basename $j); - echo "kmod($j)" - done - else - echo "kmod($kmod)" - fi -done +# Only process files from .../lib/modules/... subtree +[ "${fname#*/lib/modules/*}" != "$fname" ] || exit 0 + +kmod=${fname##*/} # like basename, but faster + +if [ "$kmod" = "modules.builtin" ]; then + for j in $(cat -- "$fname"); do + echo "kmod(${j##*/})" + done + exit 0 +fi + +kmod=${kmod%.gz} +kmod=${kmod%.xz} +if [ "${kmod%.ko}" != "$kmod" ]; then + echo "kmod($kmod)" +fi diff --git a/macros b/macros index a9f65cc..fde606a 100644 --- a/macros +++ b/macros @@ -52,12 +52,14 @@ # variables CFLAGS, CXXFLAGS, FFLAGS, FCFLAGS, LDFLAGS if they have # not been set already. RPM_OPT_FLAGS and RPM_LD_FLAGS have already # been set implicitly at the start of the %%build section. +# LT_SYS_LIBRARY_PATH is used by libtool script. %set_build_flags \ CFLAGS="${CFLAGS:-%{build_cflags}}" ; export CFLAGS ; \ CXXFLAGS="${CXXFLAGS:-%{build_cxxflags}}" ; export CXXFLAGS ; \ FFLAGS="${FFLAGS:-%{build_fflags}}" ; export FFLAGS ; \ FCFLAGS="${FCFLAGS:-%{build_fflags}}" ; export FCFLAGS ; \ - LDFLAGS="${LDFLAGS:-%{build_ldflags}}" ; export LDFLAGS + LDFLAGS="${LDFLAGS:-%{build_ldflags}}" ; export LDFLAGS ; \ + LT_SYS_LIBRARY_PATH="${LT_SYS_LIBRARY_PATH:-%_libdir:}" ; export LT_SYS_LIBRARY_PATH # Internal-only. Do not use. Expand a variable and strip the flags # not suitable to extension builders. @@ -144,9 +146,10 @@ print(result) %__brp_ldconfig /usr/lib/rpm/redhat/brp-ldconfig %__brp_compress /usr/lib/rpm/brp-compress %__brp_strip /usr/lib/rpm/brp-strip %{__strip} +%__brp_strip_lto /usr/lib/rpm/redhat/brp-strip-lto %{__strip} %__brp_strip_comment_note /usr/lib/rpm/brp-strip-comment-note %{__strip} %{__objdump} %__brp_strip_static_archive /usr/lib/rpm/brp-strip-static-archive %{__strip} -%__brp_python_bytecompile /usr/lib/rpm/brp-python-bytecompile "%{__python}" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" +%__brp_python_bytecompile /usr/lib/rpm/redhat/brp-python-bytecompile "%{__python}" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" %__brp_python_hardlink /usr/lib/rpm/brp-python-hardlink # __brp_mangle_shebangs_exclude - shebangs to exclude # __brp_mangle_shebangs_exclude_file - file from which to get shebangs to exclude @@ -161,6 +164,7 @@ print(result) %{?__brp_strip} \ %{?__brp_strip_comment_note} \ } \ + %{?__brp_strip_lto} \ %{?__brp_strip_static_archive} \ %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ %{?__brp_python_hardlink} \ @@ -228,7 +232,7 @@ print(result) %_ld_as_needed 1 %_ld_as_needed_flags %{?_ld_as_needed:-Wl,--as-needed} -%__global_compiler_flags -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches %{_hardened_cflags} %{_annotated_cflags} +%__global_compiler_flags -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches %{_hardened_cflags} %{_annotated_cflags}%{?_legacy_common_support: -fcommon} # Automatically trim changelog entries after 2 years %_changelog_trimtime %{lua:print(os.time() - 2 * 365 * 86400)} diff --git a/macros.fedora-misc b/macros.fedora-misc index 83e455a..df96ba5 100644 --- a/macros.fedora-misc +++ b/macros.fedora-misc @@ -43,4 +43,19 @@ fedora.writevars(macrofile,rpmvars) } # gpgverify verifies signed sources. There is documentation in the script. -%gpgverify %{_rpmconfigdir}/redhat/gpgverify +%gpgverify(k:s:d:) %{lua: +local script = rpm.expand("%{_rpmconfigdir}/redhat/gpgverify ") +local keyring = rpm.expand("%{-k*}") +local signature = rpm.expand("%{-s*}") +local data = rpm.expand("%{-d*}") +print(script) +if keyring ~= "" then + print(rpm.expand("--keyring='%{SOURCE" .. keyring .. "}' ")) +end +if signature ~= "" then + print(rpm.expand("--signature='%{SOURCE" .. signature .. "}' ")) +end +if data ~= "" then + print(rpm.expand("--data='%{SOURCE" .. data .. "}' ")) +end +} diff --git a/redhat-rpm-config.spec b/redhat-rpm-config.spec index 1dc0164..b288f38 100644 --- a/redhat-rpm-config.spec +++ b/redhat-rpm-config.spec @@ -6,8 +6,8 @@ Summary: Red Hat specific rpm configuration files Name: redhat-rpm-config -Version: 137 -Release: 1.2.riscv64%{?dist} +Version: 153 +Release: 1.0.riscv64%{?dist} # No version specified. License: GPL+ URL: https://src.fedoraproject.org/rpms/redhat-rpm-config @@ -48,6 +48,10 @@ Source155: macros.fedora-misc # and an echo when the mangling happens Source201: brp-mangle-shebangs +# this comes from rpm itself +# however, now we can do Fedora changes within +Source202: brp-python-bytecompile + # Dependency generator scripts (deprecated) Source300: find-provides Source301: find-provides.ksyms @@ -74,6 +78,7 @@ Source602: libsymlink.attr # BRPs Source700: brp-ldconfig +Source701: brp-strip-lto # Convenience lua functions Source800: common.lua @@ -87,6 +92,7 @@ BuildRequires: perl-generators Requires: coreutils Requires: efi-srpm-macros +Requires: fonts-srpm-macros Requires: fpc-srpm-macros Requires: ghc-srpm-macros Requires: gnat-srpm-macros @@ -95,7 +101,8 @@ Requires: nim-srpm-macros Requires: ocaml-srpm-macros Requires: openblas-srpm-macros Requires: perl-srpm-macros -Requires: python-srpm-macros +# ↓ Provides compileall2 Python module +Requires: python-srpm-macros >= 3-46 Requires: rust-srpm-macros Requires: qt5-srpm-macros @@ -202,17 +209,58 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm forge.lua %{_rpmconfigdir}/macros.d/macros.kmp %changelog -* Fri Jul 12 2019 David Abdurachmanov - 137-1.2.riscv64 -- Re-enable annobin +* Thu Feb 20 2020 Jason L Tibbitts III - 153-1 +- Add dependency on fonts-srpm-macros, as those have now been approved by FPC. -* Fri Jul 12 2019 David Abdurachmanov - 137-1.1.riscv64 -- Temporary change, disable annobin +* Thu Feb 20 2020 Jeff Law - 152-1 +- Use eu-elfclassify to only run strip on ELF relocatables + and archive libraries. -* Wed Jul 10 2019 David Abdurachmanov - 137-1.0.riscv64 -- Disable perl_default_subpackage_tests (test suite subpackage for perl packages) -- Disable %check for riscv64 -- Add -fasynchronous-unwind-tables -fstack-clash-protection to riscv64 (other - arches seem to have them now) +* Fri Feb 14 2020 Igor Raits - 151-1 +- Fixup parallel algorithm for brp-strip-lto + +* Fri Feb 14 2020 Jeff Law - 150-1 +- Strip LTO sections/symbols from installed .o/.a files + +* Thu Jan 23 2020 Jeff Law - 149-1 +- Allow conditionally adding -fcommon to CFLAGS by defining %%_legacy_common_support + +* Mon Jan 20 2020 Florian Weimer - 148-1 +- Reenable annobin after GCC 10 integration (#1792892) + +* Mon Jan 20 2020 Florian Weimer - 147-1 +- Temporarily disable annobin for GCC 10 (#1792892) + +* Thu Dec 05 2019 Denys Vlasenko - 146-1 +- kmod.prov: fix and speed it up + +* Tue Dec 03 15:48:18 CET 2019 Igor Gnatenko - 145-1 +- %%set_build_flags: define LT_SYS_LIBRARY_PATH + +* Thu Nov 21 2019 Denys Vlasenko - 144-1 +- Speed up brp-mangle-shebangs. + +* Tue Nov 05 2019 Lumír Balhar - 143-1 +- Fix brp-python-bytecompile with the new features from compileall2 +- Resolves: rhbz#1595265 + +* Fri Nov 01 2019 Miro Hrončok - 142-1 +- Fix the simple API of %%gpgverify. + +* Thu Aug 22 2019 Jason L Tibbitts III - 141-2 +- Simplify the API of %%gpgverify. + +* Thu Jul 25 2019 Richard W.M. Jones - 140-2 +- Bump version and rebuild. + +* Sat Jul 20 2019 Igor Gnatenko - 140-1 +- Fixup python-srpm-macros version + +* Wed Jul 17 2019 Lumír Balhar - 139-1 +- Use compileall2 Python module for byte-compilation in brp-python-bytecompile + +* Tue Jul 09 2019 Miro Hrončok - 138-1 +- Move brp-python-bytecompile from rpm, so we can easily adapt it * Mon Jul 08 2019 Nicolas Mailhot - 137-1 - listfiles: make it robust against all kinds of “interesting” inputs